Kaggle competition: Santander Value Prediction Challenge
embedding_matrix = np.concatenate(
 [build_matrix(tokenizer.word_index, f)for f in EMBEDDING_FILES], axis=-1)
<define_variables>
train_csv=pd.read_csv(".. /input/train.csv")
 test_csv=pd.read_csv(".. /input/test.csv" )
checkpoint_predictions = []
weights = []<train_model>
train_csv=train_csv.drop(['ID'],axis=1)
 test_id=test_csv['ID']
test_csv.drop(['ID'],axis=1,inplace=True)
 target=train_csv['target']
train_csv.drop(['target'],axis=1,inplace=True)
 target_log=np.log10(target)
 target_vals=np.log1p(target_log)
 print(train_csv.shape,test_csv.shape )
for model_idx in range(NUM_MODELS):
model = build_model(embedding_matrix, y_aux_train.shape[-1])
 for global_epoch in range(EPOCHS):
model.fit(
 x_train,
[y_train, y_aux_train],
batch_size=BATCH_SIZE,
epochs=1,
verbose=2,
sample_weight=[sample_weights.values, np.ones_like(sample_weights)],
callbacks=[
LearningRateScheduler(lambda _: 1e-3 *(0.55 ** global_epoch))
] 
)
 checkpoint_predictions.append(model.predict(x_test, batch_size=2048)[0].flatten())
 weights.append(2 ** global_epoch )<prepare_output>
forest = RandomForestRegressor(n_estimators = 1000, random_state = 42)
 forest.fit(train_csv,target_vals)
predictions = np.average(checkpoint_predictions, weights=weights, axis=0)
<save_to_csv>
importances = forest.feature_importances_
feature_importance = 100.0 *(importances / importances.max())
 sorted_idx = np.argsort(feature_importance)
 feature_names = list(train_csv.columns.values)
 feature_names_sort = [feature_names[indice] for indice in sorted_idx]
pos = np.arange(sorted_idx.shape[0])+.5
print('Top 40 features are: ')
 cols=[]
for feature in feature_names_sort[::-1][:40]:
cols.append(feature)
 print(sorted(cols))
submission = pd.DataFrame.from_dict({
'id': test_df.id,
'prediction': predictions
})
 submission.to_csv('submission.csv', index=False )<set_options>
train_data=train_csv[cols]
test_data=test_csv[cols]
warnings.filterwarnings('ignore' )<load_from_csv>
for df in [train_data,test_data]:
df["sum"]=df.sum(axis=1)
 df["median"]=df.median(axis=1)
 df["mean"]=df.mean(axis=1)
 df["std"]=df.std(axis=1)
 df["max"]=df.max(axis=1)
 df["min"]=df.min(axis=1)
 df["kurtosis"]=df.kurtosis(axis=1)
 print(train_data.shape,test_data.shape )
df_train = pd.read_csv("/kaggle/input/restaurant-revenue-prediction/train.csv.zip")
 df_test = pd.read_csv("/kaggle/input/restaurant-revenue-prediction/test.csv.zip" )<drop_column>
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb
df_train = df_train.drop("Id",axis=1 )<define_variables>
y_target=np.log1p(target)
 NUM_FOLDS = 5
def rmsle_cv(model):
kf = KFold(NUM_FOLDS, shuffle=True, random_state=42 ).get_n_splits(train_data.values)
 rmse= np.sqrt(-cross_val_score(model, train_data, y_target, scoring="neg_mean_squared_error", cv = kf))
return(rmse )
quantitative_feats = [i for i in df_train if df_train[i].dtype != np.object]
qualitative_feats = [i for i in df_train if df_train[i].dtype == np.object]
print("Quantitative_features: {}".format(quantitative_feats))
print("Qualtative_features: {}".format(qualitative_feats))<drop_column>
x_train, x_val, y_train, y_val = train_test_split(train_data, y_target, test_size=0.15, random_state=42 )
quantitative_feats.remove("revenue")
 qualitative_feats.remove("Open Date" )<define_variables>
params = {
"objective" : "regression",
"metric" : "rmse",
"num_leaves" : 50,
"max_depth" : 6,
"learning_rate" : 0.0005,
"bagging_fraction" : 0.7,
"feature_fraction" : 0.8,
"bagging_frequency" : 5,
"bagging_seed" : 42,
"verbosity" : -1,
"lambda" : 0.01,
"random_seed": 42
}
lgtrain = lgb.Dataset(x_train, label=y_train)
 lgval = lgb.Dataset(x_val, label=y_val)
 evals_result = {}
model = lgb.train(params, lgtrain, 200000,
valid_sets=[lgval],
early_stopping_rounds=1500,
verbose_eval=50,
evals_result=evals_result )
revenue = df_train["revenue"]<count_unique_values>
pred_test_y = np.expm1(model.predict(test_data, num_iteration=model.best_iteration))
discrete_feats = [feat for feat in quantitative_feats if df_train[feat].nunique() <= 25]
discrete_feats<count_unique_values>
params = {'objective': 'reg:linear',
'eval_metric': 'rmse',
'eta': 0.0005,
'max_depth':6,
'subsample': 0.7,
'colsample_bytree': 0.5,
'alpha':0,
'random_state': 42,
'silent': True}
tr_data = xgb.DMatrix(x_train,y_train)
 va_data = xgb.DMatrix(x_val,y_val)
 watchlist = [(tr_data, 'train'),(va_data, 'valid')]
model_xgb = xgb.train(params, tr_data, 200000, watchlist, maximize=False, early_stopping_rounds = 3000, verbose_eval=100 )
continuous_feats = [feat for feat in quantitative_feats if df_train[feat].nunique() > 25]
continuous_feats<count_missing_values>
dtest = xgb.DMatrix(test_data)
 xgb_pred_y = np.expm1(model_xgb.predict(dtest, ntree_limit=model_xgb.best_ntree_limit))
print(df_train.isnull().any() )<create_dataframe>
sub = pd.DataFrame()
sub['ID'] = test_id
sub['target'] = pred_test_y
sub.to_csv('submission_lgb.csv',index=False )
def calc_vif(X):
vif = pd.DataFrame()
vif["variables"] = X.columns
vif["VIF"] = [variance_inflation_factor(X.values, i)for i in range(X.shape[1])]
return(vif)
 vif = calc_vif(df_train[quantitative_feats])
 high_vif = vif[vif["VIF"] > 10].sort_values("VIF",ascending=False)
 high_vif<prepare_x_and_y>
sub = pd.DataFrame()
sub['ID'] = test_id
sub['target'] = xgb_pred_y
sub.to_csv('submission_xgb.csv',index=False )
y = df_train["revenue"]
df_train = df_train.drop("revenue",axis=1)
 df_all = pd.concat([df_train,df_test],axis=0)
 df_all = df_all.drop("Id",axis=1)
 df_all.shape<sort_values>
sub = pd.DataFrame()
sub['ID'] = test_id
sub['target'] = np.mean([pred_test_y,xgb_pred_y])
 sub.to_csv('submission_comb_xlgb.csv',index=False )