Kaggle competition: Santander Value Prediction Challenge
<SOS> metric: RMSLE Kaggle data source: santander-value-prediction-challenge<predict_on_test>
%matplotlib inline
warnings.filterwarnings('ignore' )
params = {"objective": "binary:logistic", "booster": "gbtree", "eval_metric":"auc"}
train_xgb = xgb.DMatrix(X, y)
 test_xgb = xgb.DMatrix(test)
 gbm = xgb.train(params, train_xgb, 20)
 y_pred = gbm.predict(test_xgb )<save_to_csv>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv' )
submission = pd.DataFrame({"ID":test.index, "TARGET":y_pred})
 submission.to_csv("submission.csv", index=False )<train_model>
print("Total Train Features with NaN Values = " + str(train_df.columns[train_df.isnull().sum() != 0].size))
if(train_df.columns[train_df.isnull().sum() != 0].size):
print("Features with NaN => {}".format(list(train_df.columns[train_df.isnull().sum() != 0])))
 train_df[train_df.columns[train_df.isnull().sum() != 0]].isnull().sum().sort_values(ascending = False )
remove = []
for col in training.columns:
if training[col].std() == 0:
remove.append(col)
 training.drop(remove, axis=1, inplace=True )<drop_column>
print("Total Test Features with NaN Values = " + str(test_df.columns[test_df.isnull().sum() != 0].size))
if(test_df.columns[test_df.isnull().sum() != 0].size):
print("Features with NaN => {}".format(list(test_df.columns[test_df.isnull().sum() != 0])))
 test_df[test_df.columns[test_df.isnull().sum() != 0]].isnull().sum().sort_values(ascending = False )
test.drop(remove, axis=1, inplace=True )<prepare_x_and_y>
colsToRemove = []
for col in train_df.columns:
if col != 'ID' and col != 'target':
if train_df[col].std() == 0:
colsToRemove.append(col)
 train_df.drop(colsToRemove, axis=1, inplace=True)
 test_df.drop(colsToRemove, axis=1, inplace=True)
 print("Removed `{}` Constant Columns
".format(len(colsToRemove)))
 print(colsToRemove )
X = training.iloc[:,:-1]
y = training.TARGET<predict_on_test>
%%time
def duplicate_columns(frame):
groups = frame.columns.to_series().groupby(frame.dtypes ).groups
dups = []
for t, v in groups.items() :
cs = frame[v].columns
vs = frame[v]
lcs = len(cs)
 for i in range(lcs):
ia = vs.iloc[:,i].values
for j in range(i+1, lcs):
ja = vs.iloc[:,j].values
if np.array_equal(ia, ja):
dups.append(cs[i])
 break
return dups
colsToRemove = duplicate_columns(train_df)
 print(colsToRemove )
params = {"objective": "binary:logistic", "booster": "gbtree", "eval_metric":"auc"}
train_xgb = xgb.DMatrix(X, y)
 test_xgb = xgb.DMatrix(test)
 gbm = xgb.train(params, train_xgb, 20)
 y_pred = gbm.predict(test_xgb )<save_to_csv>
train_df.drop(colsToRemove, axis=1, inplace=True)
 test_df.drop(colsToRemove, axis=1, inplace=True)
 print("Removed `{}` Duplicate Columns
".format(len(colsToRemove)))
 print(colsToRemove )
submission = pd.DataFrame({"ID":test.index, "TARGET":y_pred})
 submission.to_csv("submission.csv", index=False )<set_options>
def drop_sparse(train, test):
flist = [x for x in train.columns if not x in ['ID','target']]
for f in flist:
if len(np.unique(train[f])) <2:
train.drop(f, axis=1, inplace=True)
 test.drop(f, axis=1, inplace=True)
 return train, test
%reload_ext autoreload
%autoreload 2
%matplotlib inline 
<set_options>
%%time
train_df, test_df = drop_sparse(train_df, test_df )
def seed_everything(seed=123):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<define_variables>
X_train = train_df.drop(["ID", "target"], axis=1)
 print("rounding...")
 for i in X_train.columns:
X_train[i] = np.round(np.log1p(X_train[i]), decimals=3)
 gc.collect()
y_train = np.log1p(train_df["target"].values)
 X_test = test_df.drop(["ID"], axis=1)
 print("rounding...")
 for i in X_test.columns:
X_test[i] = np.round(np.log1p(X_test[i]), decimals=3 )
CRAWL_EMBEDDING_PATH = '.. /input/pickled-crawl300d2m-for-kernel-competitions/crawl-300d-2M.pkl'
GLOVE_EMBEDDING_PATH = '.. /input/pickled-glove840b300d-for-10sec-loading/glove.840B.300d.pkl' 
<load_pretrained>
dev_X, val_X, dev_y, val_y = train_test_split(X_train, y_train, test_size = 0.2, random_state = 42 )
NUM_MODELS = 2
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
MAX_LEN = 220
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path,'rb')as f:
emb_arr = pickle.load(f)
 return emb_arr<categorify>
def run_lgb(train_X, train_y, val_X, val_y, test_X):
params = {
"objective" : "regression",
"metric" : "rmse",
"num_leaves" : 40,
"learning_rate" : 0.005,
"bagging_fraction" : 0.6,
"feature_fraction" : 0.6,
"bagging_frequency" : 6,
"bagging_seed" : 42,
"verbosity" : -1,
"seed": 42
}
lgtrain = lgb.Dataset(train_X, label=train_y)
 lgval = lgb.Dataset(val_X, label=val_y)
 evals_result = {}
model = lgb.train(params, lgtrain, 5000,
valid_sets=[lgtrain, lgval],
early_stopping_rounds=100,
verbose_eval=150,
evals_result=evals_result)
 pred_test_y = np.expm1(model.predict(test_X, num_iteration=model.best_iteration))
return pred_test_y, model, evals_result
def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( max_features + 1, 300))
unknown_words = []
for word, i in word_index.items() :
if i <= max_features:
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
try:
embedding_matrix[i] = embedding_index[word.lower() ]
except KeyError:
try:
embedding_matrix[i] = embedding_index[word.title() ]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words<normalization>
pred_test, model, evals_result = run_lgb(dev_X, dev_y, val_X, val_y, X_test)
 print("LightGBM Training Completed..." )
def sigmoid(x):
return 1 /(1 + np.exp(-x))
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
def train_model(learn,test,output_dim,lr=0.001,
batch_size=512, n_epochs=5,
enable_checkpoint_ensemble=True):
all_test_preds = []
checkpoint_weights = [2 ** epoch for epoch in range(n_epochs)]
test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 n = len(learn.data.train_dl)
 phases = [(TrainingPhase(n ).schedule_hp('lr', lr *(0.6**(i)))) for i in range(n_epochs)]
sched = GeneralScheduler(learn, phases)
 learn.callbacks.append(sched)
 for epoch in range(n_epochs):
learn.fit(1)
 test_preds = np.zeros(( len(test), output_dim))
for i, x_batch in enumerate(test_loader):
X = x_batch[0].cuda()
y_pred = sigmoid(learn.model(X ).detach().cpu().numpy())
 test_preds[i * batch_size:(i+1)* batch_size, :] = y_pred
all_test_preds.append(test_preds)
 if enable_checkpoint_ensemble:
test_preds = np.average(all_test_preds, weights=checkpoint_weights, axis=0)
 else:
test_preds = all_test_preds[-1]
return test_preds<load_from_csv>
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub_lgb = pd.DataFrame()
sub_lgb["target"] = pred_test
sub["target"] =(sub_lgb["target"] )
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv' )<define_variables>
print(sub.head())
 sub.to_csv('sub_lgb.csv', index=False )