Kaggle competition: Santander Value Prediction Challenge
<SOS> metric: RMSLE Kaggle data source: santander-value-prediction-challenge<string_transform>
print(os.listdir(".. /input"))
warnings.filterwarnings("ignore")
 train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 test_ID = test['ID']
y_train = train['target']
y_train = np.log1p(y_train)
 train.drop("ID", axis = 1, inplace = True)
 train.drop("target", axis = 1, inplace = True)
 test.drop("ID", axis = 1, inplace = True)
 cols_with_onlyone_val = train.columns[train.nunique() == 1]
train.drop(cols_with_onlyone_val.values, axis=1, inplace=True)
 test.drop(cols_with_onlyone_val.values, axis=1, inplace=True)
 NUM_OF_DECIMALS = 32
train = train.round(NUM_OF_DECIMALS)
 test = test.round(NUM_OF_DECIMALS)
 colsToRemove = []
columns = train.columns
for i in range(len(columns)-1):
v = train[columns[i]].values
dupCols = []
for j in range(i + 1,len(columns)) :
if np.array_equal(v, train[columns[j]].values):
colsToRemove.append(columns[j])
 train.drop(colsToRemove, axis=1, inplace=True)
 test.drop(colsToRemove, axis=1, inplace=True)
 train.shape
tokenizer = text.Tokenizer(filters=CHARS_TO_REMOVE)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test))<string_transform>
NUM_OF_FEATURES = 1000
def rmsle(y, pred):
return np.sqrt(np.mean(np.power(y - pred, 2)))
 x1, x2, y1, y2 = model_selection.train_test_split(
 train, y_train.values, test_size=0.20, random_state=5)
 model = ensemble.RandomForestRegressor(n_jobs=-1, random_state=7)
 model.fit(x1, y1)
 print(rmsle(y2, model.predict(x2)))
 col = pd.DataFrame({'importance': model.feature_importances_, 'feature': train.columns} ).sort_values(
 by=['importance'], ascending=[False])[:NUM_OF_FEATURES]['feature'].values
train = train[col]
test = test[col]
train.shape
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )<define_variables>
THRESHOLD_P_VALUE = 0.01
THRESHOLD_STATISTIC = 0.3
diff_cols = []
for col in train.columns:
statistic, pvalue = ks_2samp(train[col].values, test[col].values)
 if pvalue <= THRESHOLD_P_VALUE and np.abs(statistic)> THRESHOLD_STATISTIC:
diff_cols.append(col)
 for col in diff_cols:
if col in train.columns:
train.drop(col, axis=1, inplace=True)
 test.drop(col, axis=1, inplace=True)
 train.shape
EMBEDDING_FILES = [
'.. /input/fast-track-300d-2m-vec/crawl-300d-2M.vec',
'.. /input/glove840b300dtxt/glove.840B.300d.txt'
]<concatenate>
ntrain = len(train)
 ntest = len(test)
 tmp = pd.concat([train,test])
 weight =(( train != 0 ).sum() /len(train)).values
tmp_train = train[train!=0]
tmp_test = test[test!=0]
train["weight_count"] =(tmp_train*weight ).sum(axis=1)
 test["weight_count"] =(tmp_test*weight ).sum(axis=1)
 train["count_not0"] =(train != 0 ).sum(axis=1)
 test["count_not0"] =(test != 0 ).sum(axis=1)
 train["sum"] = train.sum(axis=1)
 test["sum"] = test.sum(axis=1)
 train["var"] = tmp_train.var(axis=1)
 test["var"] = tmp_test.var(axis=1)
 train["median"] = tmp_train.median(axis=1)
 test["median"] = tmp_test.median(axis=1)
 train["mean"] = tmp_train.mean(axis=1)
 test["mean"] = tmp_test.mean(axis=1)
 train["std"] = tmp_train.std(axis=1)
 test["std"] = tmp_test.std(axis=1)
 train["max"] = tmp_train.max(axis=1)
 test["max"] = tmp_test.max(axis=1)
 train["min"] = tmp_train.min(axis=1)
 test["min"] = tmp_test.min(axis=1)
 train["skew"] = tmp_train.skew(axis=1)
 test["skew"] = tmp_test.skew(axis=1)
 train["kurtosis"] = tmp_train.kurtosis(axis=1)
 test["kurtosis"] = tmp_test.kurtosis(axis=1)
 del(tmp_train)
 del(tmp_test)
 NUM_OF_COM = 100
transformer = random_projection.SparseRandomProjection(n_components = NUM_OF_COM)
 RP = transformer.fit_transform(tmp)
 rp = pd.DataFrame(RP)
 columns = ["RandomProjection{}".format(i)for i in range(NUM_OF_COM)]
rp.columns = columns
rp_train = rp[:ntrain]
rp_test = rp[ntrain:]
rp_test.index = test.index
train = pd.concat([train,rp_train],axis=1)
 test = pd.concat([test,rp_test],axis=1)
 del(rp_train)
 del(rp_test)
 train.shape
embedding_matrix = np.concatenate(
 [build_matrix(tokenizer.word_index, f)for f in EMBEDDING_FILES], axis=-1 )<define_variables>
NUM_FOLDS = 5
def rmsle_cv(model):
kf = KFold(NUM_FOLDS, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse= np.sqrt(-cross_val_score(model, train, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse)
 class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([ model.predict(X)for model in self.models_ ])
 return np.mean(predictions, axis=1)
 model_xgb = xgb.XGBRegressor(colsample_bytree=0.055, colsample_bylevel =0.5,
gamma=1.5, learning_rate=0.02, max_depth=32,
objective='reg:linear',booster='gbtree',
min_child_weight=57, n_estimators=1000, reg_alpha=0,
reg_lambda = 0,eval_metric = 'rmse', subsample=0.7,
silent=1, n_jobs = -1, early_stopping_rounds = 14,
random_state =7, nthread = -1)
 model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=144,
learning_rate=0.005, n_estimators=720, max_depth=13,
metric='rmse',is_training_metric=True,
max_bin = 55, bagging_fraction = 0.8,verbose=-1,
bagging_freq = 5, feature_fraction = 0.9)
 score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(model_lgb)
 print("LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
averaged_models = AveragingModels(models =(model_xgb, model_lgb))
score = rmsle_cv(averaged_models)
 print("averaged score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
<train_model><EOS> 
averaged_models.fit(train.values, y_train)
 pred = np.expm1(averaged_models.predict(test.values))
ensemble = pred
sub = pd.DataFrame()
sub['ID'] = test_ID
sub['target'] = ensemble
sub.to_csv('submission.csv',index=False)