Kaggle competition: TMDB Box Office Prediction
<SOS> metric: RMSLE Kaggle data source: tmdb-box-office-prediction<set_options>
warnings.simplefilter(action='ignore', category=FutureWarning)
 %matplotlib inline
seed = 123
aml.leader<save_to_csv>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 print('train dataset size:', train.shape)
 print('test dataset size:', test.shape )
preds = aml.predict(test)
 sample_submission = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv')
 sample_submission['time_to_failure'] = preds.as_data_frame().values.flatten()
sample_submission.to_csv('h2o_submission_fold0.csv', index=False )<train_model>
columns_to_keep = set()
%%time
nfolds=5
aml = H2OAutoML(max_models=500, seed=554, max_runtime_secs=8000,nfolds=nfolds,sort_metric="MAE")
 aml.train(x=x, y=y, training_frame=train )<set_options>
def clean_dictionary_features(feature_name,train,test):
train[feature_name] = train[feature_name].apply(lambda x:{} if pd.isna(x)else ast.literal_eval(x))
test[feature_name] = test[feature_name].apply(lambda x:{} if pd.isna(x)else ast.literal_eval(x))
train[feature_name+'_number'] = train[feature_name].apply(lambda x:len(x)if x!={} else 0)
 test[feature_name+'_number'] = test[feature_name].apply(lambda x:len(x)if x!={} else 0)
 columns_to_keep.add(feature_name+'_number')
 list_of_values = list(train[feature_name].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 train[feature_name+'_all'] = train[feature_name].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_values = [m[0] for m in Counter([i for j in list_of_values for i in j] ).most_common(10)]
for val in top_values:
train[feature_name +'_'+val] = train[feature_name+'_all'].apply(lambda x: 1 if val in x else 0)
 columns_to_keep.add(feature_name +'_'+val)
 test[feature_name+'_all'] = test[feature_name].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for val in top_values:
test[feature_name +'_'+val] = test[feature_name+'_all'].apply(lambda x: 1 if val in x else 0)
 le = LabelEncoder()
le.fit(list(train[feature_name+'_all'].fillna('')) + list(test[feature_name+'_all'].fillna('')))
 train[feature_name+'_all'] = le.transform(train[feature_name+'_all'].fillna('' ).astype(str))
test[feature_name+'_all'] = le.transform(test[feature_name+'_all'].fillna('' ).astype(str))
columns_to_keep.add(feature_name+'_all')
 return train,test
aml.leader<save_to_csv>
train,test = clean_dictionary_features('genres',train,test)
 train,test = clean_dictionary_features('production_companies',train,test)
 train,test = clean_dictionary_features('production_countries',train,test)
 train,test = clean_dictionary_features('spoken_languages',train,test)
 train,test = clean_dictionary_features('Keywords',train,test)
 train,test = clean_dictionary_features('cast',train,test)
 train,test = clean_dictionary_features('crew',train,test)
preds = aml.predict(test)
 sample_submission = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv')
 sample_submission['time_to_failure'] = preds.as_data_frame().values.flatten()
sample_submission.to_csv('h2o_submission_fold5.csv', index=False )<save_to_csv>
print("-"*40,'
',train.isnull().sum())
 print("-"*40,'
',test.isnull().sum() )
sample_submission = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv')
 sample_submission['time_to_failure'] =(pd.read_csv('h2o_submission_fold0.csv')['time_to_failure']+pd.read_csv('h2o_submission_fold5.csv')['time_to_failure'])/2
sample_submission.to_csv('h2o_submission_blend.csv', index=False )<set_options>
train['got_collection'] = train['belongs_to_collection'].apply(lambda x:0 if pd.isnull(x)else 1)
 test['got_collection'] = test['belongs_to_collection'].apply(lambda x:0 if pd.isnull(x)else 1 )
sns.set()<define_variables>
len(train[train['budget']==0] )
warnings.filterwarnings('ignore')
 NUM_EVALS = 1000
N_FOLDS = 5
LGBM_MAX_LEAVES = 2**10
LGBM_MAX_DEPTH = 25
EVAL_METRIC_LGBM_REG = 'mae'
EVAL_METRIC_LGBM_CLASS = 'auc'
XGB_MAX_LEAVES = 2**12
XGB_MAX_DEPTH = 25
EVAL_METRIC_XGB_REG = 'mae'
EVAL_METRIC_XGB_CLASS = 'auc'
CB_MAX_DEPTH = 8
OBJECTIVE_CB_REG = 'MAE'
OBJECTIVE_CB_CLASS = 'Logloss'
BEST_SCORE = 0
def quick_hyperopt(data, labels, package='lgbm', num_evals=NUM_EVALS, diagnostic=False, Class=False):
if package=='lgbm':
print('Running {} rounds of LightGBM parameter optimisation:'.format(num_evals))
gc.collect()
integer_params = ['max_depth',
'num_leaves',
'max_bin',
'min_data_in_leaf',
'min_data_in_bin']
def objective(space_params):
for param in integer_params:
space_params[param] = int(space_params[param])
 if space_params['boosting']['boosting'] == 'goss':
top_rate = space_params['boosting'].get('top_rate')
 other_rate = space_params['boosting'].get('other_rate')
 top_rate = max(top_rate, 0)
 top_rate = min(top_rate, 0.5)
 other_rate = max(other_rate, 0)
 other_rate = min(other_rate, 0.5)
 space_params['top_rate'] = top_rate
space_params['other_rate'] = other_rate
subsample = space_params['boosting'].get('subsample', 1.0)
 space_params['boosting'] = space_params['boosting']['boosting']
space_params['subsample'] = subsample
if Class:
cv_results = lgb.cv(space_params, train, nfold = N_FOLDS, stratified=True,
early_stopping_rounds=100, metrics=EVAL_METRIC_LGBM_CLASS, seed=42)
 best_loss = 1 - cv_results['auc-mean'][-1]
else:
cv_results = lgb.cv(space_params, train, nfold = N_FOLDS, stratified=False,
early_stopping_rounds=100, metrics=EVAL_METRIC_LGBM_REG, seed=42)
 best_loss = cv_results['l1-mean'][-1]
return{'loss':best_loss, 'status': STATUS_OK }
train = lgb.Dataset(data, labels)
 boosting_list = [{'boosting': 'gbdt',
'subsample': hp.uniform('subsample', 0.5, 1)},
{'boosting': 'goss',
'subsample': 1.0,
'top_rate': hp.uniform('top_rate', 0, 0.5),
'other_rate': hp.uniform('other_rate', 0, 0.5)}]
if Class:
metric_list = ['auc']
objective_list = ['binary', 'cross_entropy']
else:
metric_list = ['MAE', 'RMSE']
objective_list = ['huber', 'gamma',
'fair', 'tweedie']
space ={'boosting' : hp.choice('boosting', boosting_list),
'num_leaves' : hp.quniform('num_leaves', 2, LGBM_MAX_LEAVES, 1),
'max_depth': hp.quniform('max_depth', 2, LGBM_MAX_DEPTH, 1),
'max_bin': hp.quniform('max_bin', 32, 255, 1),
'min_data_in_leaf': hp.quniform('min_data_in_leaf', 1, 256, 1),
'min_data_in_bin': hp.quniform('min_data_in_bin', 1, 256, 1),
'min_gain_to_split' : hp.quniform('min_gain_to_split', 0.1, 5, 0.01),
'lambda_l1' : hp.uniform('lambda_l1', 0, 5),
'lambda_l2' : hp.uniform('lambda_l2', 0, 5),
'learning_rate' : hp.loguniform('learning_rate', np.log(0.005), np.log(0.2)) ,
'metric' : hp.choice('metric', metric_list),
'objective' : hp.choice('objective', objective_list),
'feature_fraction' : hp.quniform('feature_fraction', 0.5, 1, 0.01),
'bagging_fraction' : hp.quniform('bagging_fraction', 0.5, 1, 0.01)
 }
trials = Trials()
best = fmin(fn=objective,
space=space,
algo=tpe.suggest,
max_evals=num_evals,
trials=trials)
 best['boosting'] = boosting_list[best['boosting']]['boosting']
best['metric'] = metric_list[best['metric']]
best['objective'] = objective_list[best['objective']]
for param in integer_params:
best[param] = int(best[param])
 print('{' + '
'.join('{}: {}'.format(k, v)for k, v in best.items())+ '}')
 if diagnostic:
return(best, trials)
 else:
return(best)
 if package=='xgb':
print('Running {} rounds of XGBoost parameter optimisation:'.format(num_evals))
gc.collect()
integer_params = ['max_depth']
def objective(space_params):
for param in integer_params:
space_params[param] = int(space_params[param])
 if space_params['tree_method']['tree_method'] == 'hist':
max_bin = space_params['tree_method'].get('max_bin')
 space_params['max_bin'] = int(max_bin)
 if space_params['tree_method']['grow_policy']['grow_policy']['grow_policy'] == 'depthwise':
grow_policy = space_params['tree_method'].get('grow_policy' ).get('grow_policy' ).get('grow_policy')
 space_params['grow_policy'] = grow_policy
space_params['tree_method'] = 'hist'
else:
max_leaves = space_params['tree_method']['grow_policy']['grow_policy'].get('max_leaves')
 space_params['grow_policy'] = 'lossguide'
space_params['max_leaves'] = int(max_leaves)
 space_params['tree_method'] = 'hist'
else:
space_params['tree_method'] = space_params['tree_method'].get('tree_method')
 cv_results = xgb.cv(space_params, train, nfold=N_FOLDS, metrics=[EVAL_METRIC_XGB_REG],
early_stopping_rounds=100, stratified=False, seed=42)
 best_loss = cv_results['test-mae-mean'].iloc[-1]
return{'loss':best_loss, 'status': STATUS_OK }
train = xgb.DMatrix(data, labels)
 boosting_list = ['gbtree', 'gblinear']
metric_list = ['MAE', 'RMSE']
tree_method = [{'tree_method' : 'exact'},
{'tree_method' : 'approx'},
{'tree_method' : 'hist',
'max_bin': hp.quniform('max_bin', 2**3, 2**7, 1),
'grow_policy' : {'grow_policy': {'grow_policy':'depthwise'},
'grow_policy' : {'grow_policy':'lossguide',
'max_leaves': hp.quniform('max_leaves', 32, XGB_MAX_LEAVES, 1)}}}]
objective_list_reg = ['reg:linear', 'reg:gamma', 'reg:tweedie']
objective_list_class = ['reg:logistic', 'binary:logistic']
objective_list = objective_list_reg
space ={'boosting' : hp.choice('boosting', boosting_list),
'tree_method' : hp.choice('tree_method', tree_method),
'max_depth': hp.quniform('max_depth', 2, XGB_MAX_DEPTH, 1),
'reg_alpha' : hp.uniform('reg_alpha', 0, 5),
'reg_lambda' : hp.uniform('reg_lambda', 0, 5),
'min_child_weight' : hp.uniform('min_child_weight', 0, 5),
'gamma' : hp.uniform('gamma', 0, 5),
'learning_rate' : hp.loguniform('learning_rate', np.log(0.005), np.log(0.2)) ,
'eval_metric' : hp.choice('eval_metric', metric_list),
'objective' : hp.choice('objective', objective_list),
'colsample_bytree' : hp.quniform('colsample_bytree', 0.1, 1, 0.01),
'colsample_bynode' : hp.quniform('colsample_bynode', 0.1, 1, 0.01),
'colsample_bylevel' : hp.quniform('colsample_bylevel', 0.1, 1, 0.01),
'subsample' : hp.quniform('subsample', 0.5, 1, 0.05),
'nthread' : -1
}
trials = Trials()
best = fmin(fn=objective,
space=space,
algo=tpe.suggest,
max_evals=num_evals,
trials=trials)
 best['tree_method'] = tree_method[best['tree_method']]['tree_method']
best['boosting'] = boosting_list[best['boosting']]
best['eval_metric'] = metric_list[best['eval_metric']]
best['objective'] = objective_list[best['objective']]
for param in integer_params:
best[param] = int(best[param])
 if 'max_leaves' in best:
best['max_leaves'] = int(best['max_leaves'])
 if 'max_bin' in best:
best['max_bin'] = int(best['max_bin'])
 print('{' + '
'.join('{}: {}'.format(k, v)for k, v in best.items())+ '}')
 if diagnostic:
return(best, trials)
 else:
return(best)
 if package=='cb':
print('Running {} rounds of CatBoost parameter optimisation:'.format(num_evals))
gc.collect()
integer_params = ['depth',
'min_data_in_leaf',
'max_bin']
def objective(space_params):
for param in integer_params:
space_params[param] = int(space_params[param])
 if space_params['bootstrap_type']['bootstrap_type'] == 'Bayesian':
bagging_temp = space_params['bootstrap_type'].get('bagging_temperature')
 space_params['bagging_temperature'] = bagging_temp
if space_params['grow_policy']['grow_policy'] == 'LossGuide':
max_leaves = space_params['grow_policy'].get('max_leaves')
 space_params['max_leaves'] = int(max_leaves)
 space_params['bootstrap_type'] = space_params['bootstrap_type']['bootstrap_type']
space_params['grow_policy'] = space_params['grow_policy']['grow_policy']
space_params['random_strength'] = max(space_params['random_strength'], 0)
 space_params['fold_len_multiplier'] = max(space_params['fold_len_multiplier'], 1)
 cv_results = cb.cv(train, space_params, fold_count=N_FOLDS,
early_stopping_rounds=25, stratified=False, partition_random_seed=42)
 best_loss = cv_results['test-MAE-mean'].iloc[-1]
return{'loss':best_loss, 'status': STATUS_OK}
train = cb.Pool(data, labels.astype('float32'))
bootstrap_type = [{'bootstrap_type':'Poisson'},
{'bootstrap_type':'Bayesian',
'bagging_temperature' : hp.loguniform('bagging_temperature', np.log(1), np.log(50)) },
{'bootstrap_type':'Bernoulli'}]
LEB = ['No', 'AnyImprovement', 'Armijo']
grow_policy = [{'grow_policy':'SymmetricTree'},
{'grow_policy':'Depthwise'},
{'grow_policy':'Lossguide',
'max_leaves': hp.quniform('max_leaves', 2, 32, 1)}]
eval_metric_list_reg = ['MAE', 'RMSE', 'Poisson']
eval_metric_list_class = ['Logloss', 'AUC', 'F1']
eval_metric_list = eval_metric_list_reg
space ={'depth': hp.quniform('depth', 2, CB_MAX_DEPTH, 1),
'max_bin' : hp.quniform('max_bin', 1, 32, 1),
'l2_leaf_reg' : hp.uniform('l2_leaf_reg', 0, 5),
'min_data_in_leaf' : hp.quniform('min_data_in_leaf', 1, 50, 1),
'random_strength' : hp.loguniform('random_strength', np.log(0.005), np.log(5)) ,
'bootstrap_type' : hp.choice('bootstrap_type', bootstrap_type),
'learning_rate' : hp.uniform('learning_rate', 0.05, 0.25),
'eval_metric' : hp.choice('eval_metric', eval_metric_list),
'objective' : OBJECTIVE_CB_REG,
'leaf_estimation_backtracking' : hp.choice('leaf_estimation_backtracking', LEB),
'grow_policy': hp.choice('grow_policy', grow_policy),
'fold_len_multiplier' : hp.loguniform('fold_len_multiplier', np.log(1.01), np.log(2.5)) ,
'od_type' : 'Iter',
'od_wait' : 25,
'task_type' : 'GPU',
'verbose' : 0
}
trials = Trials()
best = fmin(fn=objective,
space=space,
algo=tpe.suggest,
max_evals=num_evals,
trials=trials)
 best['bootstrap_type'] = bootstrap_type[best['bootstrap_type']]['bootstrap_type']
best['grow_policy'] = grow_policy[best['grow_policy']]['grow_policy']
best['eval_metric'] = eval_metric_list[best['eval_metric']]
best['leaf_estimation_backtracking'] = LEB[best['leaf_estimation_backtracking']]
for param in integer_params:
best[param] = int(best[param])
 if 'max_leaves' in best:
best['max_leaves'] = int(best['max_leaves'])
 print('{' + '
'.join('{}: {}'.format(k, v)for k, v in best.items())+ '}')
 if diagnostic:
return(best, trials)
 else:
return(best)
 else:
print('Package not recognised.Please use "lgbm" for LightGBM, "xgb" for XGBoost or "cb" for CatBoost.' )<split>
train['budget_log'] = np.log1p(train.budget.values)
 test['budget_log'] = np.log1p(test.budget.values )
def quick_kfold_imp(X, y, test=None, params=None, n_fold=6, random_state=1127):
folds = KFold(n_splits=n_fold, shuffle=True, random_state=random_state)
 test_preds = np.zeros(len(test))
oof_preds = np.zeros(len(X))
imp_s = np.zeros(X.shape[1])
 imp_g = np.zeros(X.shape[1])
 if type(y)is not np.ndarray:
y = y.values.flatten()
for train_idx, valid_idx in folds.split(y):
X_train, X_valid = X.iloc[train_idx, :], X.iloc[valid_idx, :]
y_train, y_valid = y[train_idx], y[valid_idx]
model = lgb.LGBMRegressor(**params, n_estimators = 50000, n_jobs = -1, eval_metric='mae', importance_type='split')
 model.fit(X_train, y_train,eval_set=[(X_train, y_train),(X_valid, y_valid)],
verbose=0, early_stopping_rounds=200)
 val_preds = model.predict(X_valid)
 imp_s += model.feature_importances_/n_fold
model = lgb.LGBMRegressor(**params,
n_estimators = 50000,
n_jobs = -1, eval_metric='mae',
importance_type='gain')
 model.fit(X_train, y_train,eval_set=[(X_train, y_train),(X_valid, y_valid)],
verbose=0, early_stopping_rounds=200)
 oof_preds[valid_idx] = model.predict(X_valid)
 imp_g += model.feature_importances_/n_fold
test_preds += model.predict(test)/n_fold
print('OOF MAE: {:.7f}'.format(mean_absolute_error(y[:NUM_SEGMENTS],
oof_preds[:NUM_SEGMENTS])))
 return imp_s, imp_g, test_preds, oof_preds<load_from_csv>
columns_to_keep.add('budget_log' )
_train = pd.read_csv('.. /input/lanl-earthquake-nonmagic-features/train_X.csv')
 _test = pd.read_csv('.. /input/lanl-earthquake-nonmagic-features/test_X.csv')
 y = pd.read_csv('.. /input/lanl-features/y.csv' ).values.flatten() [:-1]<define_variables>
train['belongs_to_three_lang'] = train.original_language.apply(lambda x:1 if str(x)in['en','zh','tr'] else 0)
 test['belongs_to_three_lang'] = test.original_language.apply(lambda x:1 if str(x)in['en','zh','tr'] else 0 )
train = _train
test = _test
NUM_SEGMENTS = len(y )<define_variables>
columns_to_keep.add('belongs_to_three_lang' )
refined_set = ['abs_max_roll_mean_1000',
'abs_min',
'abs_q01',
'abs_trend',
'autocorrelation_1000',
'av_change_abs_roll_std_10',
'avg_first_10000',
'avg_first_50000',
'c3_5',
'classic_sta_lta4_mean',
'count_big_50000_threshold_5',
'energy_spectra_22640hz',
'energy_spectra_9306hz',
'energy_spectra_lowest100_denoised',
'energy_spectra_norm_109306hz',
'energy_spectra_norm_149306hz_denoised',
'energy_spectra_norm_15973hz_denoised',
'energy_spectra_norm_169306hz_denoised',
'energy_spectra_norm_22640hz_denoised',
'energy_spectra_norm_29306hz_denoised',
'energy_spectra_norm_49306hz_denoised',
'energy_spectra_norm_62640hz_denoised',
'energy_spectra_norm_69306hz',
'energy_spectra_norm_89306hz_denoised',
'energy_spectra_norm_9306hz_denoised',
'energy_spectra_norm_95973hz_denoised',
'fft_100_roll_std_70',
'fft_mean_change_rate',
'fft_min_roll_mean_100',
'fft_spkt_welch_density_100',
'fft_spkt_welch_density_5',
'fft_time_rev_asym_stat_10',
'iqr',
'kstat_3',
'kurt',
'mad',
'max_first_5000',
'max_last_10000',
'max_to_min',
'mean',
'med',
'mean_change_abs',
'num_crossings',
'num_peaks_10',
'q01_roll_std_1000',
'q05_roll_mean_100',
'q05_roll_std_1000',
'q95_roll_mean_100',
'q99_roll_mean_1000',
'skew',
'std_0_to_10',
'std_neg_10_to_0',
'std_neg_2_to_2']<drop_column>
le = LabelEncoder()
le.fit(list(train['original_language'].fillna('')) + list(test['original_language'].fillna('')))
 train['original_language_encoded'] = le.transform(train['original_language'].fillna('' ).astype(str))
test['original_language_encoded'] = le.transform(test['original_language'].fillna('' ).astype(str))
train = _train[refined_set]
test = _test[refined_set] 
<init_hyperparams>
columns_to_keep.add('original_language_encoded' )
ref_params = {'bagging_fraction': 0.71,
'boosting': 'gbdt',
'feature_fraction': 0.76,
'lambda_l1': 3.131216244016188,
'lambda_l2': 2.4124061313905836,
'learning_rate': 0.049886848207269734,
'max_bin': 193,
'max_depth': 15,
'metric': 'MAE',
'min_data_in_bin': 167,
'min_data_in_leaf': 62,
'min_gain_to_split': 2.07,
'num_leaves': 38,
'objective': 'fair',
'subsample': 0.9133120405819966}<train_model>
train['original_title_length'] = train.original_title.str.len()
test['original_title_length'] = test.original_title.str.len()
lgbm_params = quick_hyperopt(train, y, 'lgbm', 150 )<compute_train_metric>
def clean_date(date):
year = date.split('/')[2]
if int(year)<= 19:
return date[:-2] + '20' + year
else:
return date[:-2] + '19' + year
train_features = train.columns
imp_split = np.zeros(train.shape[1])
 imp_gain = np.zeros(train.shape[1])
 test_preds = np.zeros(len(test))
train_preds = np.zeros(len(train))
N = 30
for i in tqdm_notebook(range(N)) :
imp_s, imp_g, preds, oof_preds = quick_kfold_imp(train, y, test,
lgbm_params, random_state=i)
 imp_split += imp_s/N
imp_gain += imp_g/N
test_preds += preds/N
train_preds += oof_preds/N
initial_imp = pd.DataFrame({'feature':train_features,
'importance_split':imp_split,
'importance_gain':imp_gain,
'importance_score':np.log(( imp_split*imp_gain)) })
 initial_imp.sort_values('importance_score', ascending=False, inplace=True)
 initial_imp.head(10 )<define_variables>
test.loc[test['release_date'].isnull() == True, 'release_date'] = '05/01/00'
index1 =(y>11)
 index2 = np.logical_and(y<11, y>9)
 index3 = np.logical_and(y<9, y>7)
 index4 = np.logical_and(y<7, y>5)
 index5 = np.logical_and(y<5, y>3)
 index6 = np.logical_and(y<3, y>1)
 index7 =(y<1 )<compute_test_metric>
train['release_date'] = train['release_date'].apply(lambda x:clean_date(x))
test['release_date'] = test['release_date'].apply(lambda x:clean_date(x))
train['release_date'] = pd.to_datetime(train['release_date'])
 test['release_date'] = pd.to_datetime(test['release_date'] )
print("OOF MAE for different time_to_failure:")
 print("MAE for ttf>11 = {:.5f}".format(mean_absolute_error(y[index1], train_preds[index1])))
 print("MAE for 9<ttf<11 = {:.5f}".format(mean_absolute_error(y[index2], train_preds[index2])))
 print("MAE for 7<ttf<9 = {:.5f}".format(mean_absolute_error(y[index3], train_preds[index3])))
 print("MAE for 5<ttf<7 = {:.5f}".format(mean_absolute_error(y[index4], train_preds[index4])))
 print("MAE for 3<ttf<5 = {:.5f}".format(mean_absolute_error(y[index5], train_preds[index5])))
 print("MAE for 1<ttf<3 = {:.5f}".format(mean_absolute_error(y[index6], train_preds[index6])))
 print("MAE for ttf<1 = {:.5f}".format(mean_absolute_error(y[index7], train_preds[index7])) )<features_selection>
def date_features(dataset):
date_sections = ["year", "weekday", "month", 'weekofyear', 'day']
for sec in date_sections:
section_col = 'release_date' + "_" + sec
dataset[section_col] = getattr(dataset['release_date'].dt, sec ).astype(int)
 columns_to_keep.add(section_col)
 return dataset
imp_idx = list(initial_imp.feature.values[:50] )<sort_values>
train = date_features(train)
 test = date_features(test )
np.abs(train[imp_idx].corrwith(train['energy_spectra_norm_89306hz_denoised'])) \ 
.sort_values(ascending=False)[:10]<sort_values>
train.runtime.isnull().sum()
np.abs(train[imp_idx].corrwith(train['num_peaks_10'])).sort_values(ascending=False)[:10]<sort_values>
test.runtime.isnull().sum()
np.abs(train[imp_idx].corrwith(train['fft_100_roll_std_70'])).sort_values(ascending=False)[:10]<save_to_csv>
train.runtime = train.runtime.fillna(np.mean(train.runtime))
test.runtime = test.runtime.fillna(np.mean(test.runtime))
sub_1 = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv')
 sub_1['time_to_failure'] = test_preds
sub_1.to_csv('sub_orginal_data.csv', index=False )<define_variables>
train.status.value_counts()
a = np.arange(0, train.shape[1])
 train_aug = pd.DataFrame(index=train_select.index, columns=train.columns, dtype='float64')
 for i in tqdm_notebook(range(len(train_select))):
AUG_FEATURE_RATIO = 0.6
AUG_FEATURE_COUNT = np.floor(train_select.shape[1]*AUG_FEATURE_RATIO ).astype('int16')
 aug_feature_index = np.random.choice(train_select.shape[1], AUG_FEATURE_COUNT, replace=False)
 aug_feature_index.sort()
feature_index = np.where(np.logical_not(np.in1d(a, aug_feature_index)))[0]
train_aug.iloc[i, feature_index] = train_select.iloc[i, feature_index]
rand_row_index = np.random.choice(len(train_select), len(aug_feature_index), replace=True)
 for n, j in enumerate(aug_feature_index):
train_aug.iloc[i, j] = train_select.iloc[rand_row_index[n], j]<train_model>
train['tagline_count'] = train['tagline'].apply(lambda x: 0 if pd.isnull(x)else len(x))
test['tagline_count'] = test['tagline'].apply(lambda x: 0 if pd.isnull(x)else len(x))
params_all = quick_hyperopt(train_all, y_all, 'lgbm', 150 )<define_variables>
columns_to_keep.add('tagline_count' )
imp_split_all = np.zeros(train_aug.shape[1])
 imp_gain_all = np.zeros(train_aug.shape[1])
 test_preds_aug = np.zeros(len(test))
train_preds_aug = np.zeros(len(train_all))
N = 30
for i in tqdm_notebook(range(N)) :
a = np.arange(0, train_select.shape[1])
 np.random.seed(i)
 train_aug = pd.DataFrame(index=train_select.index, columns=train.columns, dtype='float64')
 for i in range(0, len(train_select)) :
AUG_FEATURE_RATIO = 0.6
AUG_FEATURE_COUNT = np.floor(train_select.shape[1]*AUG_FEATURE_RATIO ).astype('int16')
 aug_feature_index = np.random.choice(train_select.shape[1], AUG_FEATURE_COUNT, replace=False)
 aug_feature_index.sort()
feature_index = np.where(np.logical_not(np.in1d(a, aug_feature_index)))[0]
train_aug.iloc[i, feature_index] = train_select.iloc[i, feature_index]
rand_row_index = np.random.choice(len(train_select), len(aug_feature_index), replace=True)
 for n, j in enumerate(aug_feature_index):
train_aug.iloc[i, j] = train_select.iloc[rand_row_index[n], j]
train_all = pd.concat([train, train_aug])
 imp_s, imp_g, preds, oof_preds = quick_kfold_imp(train_all, y_all, test,
params=params_all, random_state=i)
 imp_split_all += imp_s/N
imp_gain_all += imp_g/N
test_preds_aug += preds/N
train_preds_aug += oof_preds/N 
<compute_test_metric>
train['title_count'] = train['title'].apply(lambda x: 0 if pd.isnull(x)else len(x))
test['title_count'] = test['title'].apply(lambda x: 0 if pd.isnull(x)else len(x))
_train_preds_aug = train_preds_aug[:NUM_SEGMENTS]
print("OOF mae for different time_to_failure after data augmentation:")
 print("MAE for ttf>11 = {:.5f}".format(mean_absolute_error(y[index1], _train_preds_aug[index1])))
 print("MAE for 9<ttf<11 = {:.5f}".format(mean_absolute_error(y[index2], _train_preds_aug[index2])))
 print("MAE for 7<ttf<9 = {:.5f}".format(mean_absolute_error(y[index3], _train_preds_aug[index3])))
 print("MAE for 5<ttf<7 = {:.5f}".format(mean_absolute_error(y[index4], _train_preds_aug[index4])))
 print("MAE for 3<ttf<5 = {:.5f}".format(mean_absolute_error(y[index5], _train_preds_aug[index5])))
 print("MAE for 1<ttf<3 = {:.5f}".format(mean_absolute_error(y[index6], _train_preds_aug[index6])))
 print("MAE for ttf<1 = {:.5f}".format(mean_absolute_error(y[index7], _train_preds_aug[index7])) )<create_dataframe>
train['budget_popularity'] = train['budget']/train['popularity']
test['budget_popularity'] = test['budget']/test['popularity']
columns_to_keep.add('budget_popularity')
 train['budget_release_year'] = train['budget']/train['release_date_year']
test['budget_release_year'] = test['budget']/test['release_date_year']
columns_to_keep.add('budget_release_year')
 train['popularity_release_year'] = train['popularity']/train['release_date_year']
test['popularity_release_year'] = test['popularity']/test['release_date_year']
columns_to_keep.add('popularity_release_year')
 train['popularity_release_weekday'] = np.sqrt(train['popularity']*train['release_date_weekday'])
 test['popularity_release_weekday'] = np.sqrt(test['popularity']*test['release_date_weekday'])
 columns_to_keep.add('popularity_release_weekday')
 train['genres_number_release_year'] = train['genres_number']/train['release_date_year']
test['genres_number_release_year'] = test['genres_number']/test['release_date_year']
columns_to_keep.add('genres_number_release_year')
 train['runtime_release_year'] = np.sqrt(train['runtime']*train['release_date_year'])
 test['runtime_release_year'] = np.sqrt(test['runtime']*test['release_date_year'])
 columns_to_keep.add('runtime_release_year')
 train['budget_runtime'] = np.sqrt(train['budget']*train['runtime'])
 test['budget_runtime'] = np.sqrt(test['budget']*test['runtime'])
 columns_to_keep.add('budget_runtime')
 train['budget_tagline_count'] = np.sqrt(train['budget']*train['tagline_count'])
 test['budget_tagline_count'] = np.sqrt(test['budget']*test['tagline_count'])
 columns_to_keep.add('budget_tagline_count' )
aug_imp = pd.DataFrame({'feature':train_features,
'importance_split':imp_split_all,
'importance_gain':imp_gain_all,
'importance_score':np.log(( imp_split_all*imp_gain_all)) } )<sort_values>
numerical_features = ['Keywords_number','runtime','spoken_languages_number','production_countries_number',
'production_companies_number','popularity','genres_number','crew_number','cast_number','budget_log',
'budget_popularity','budget_release_year','popularity_release_year','popularity_release_weekday',
'genres_number_release_year','runtime_release_year','budget_runtime','budget_tagline_count']
date_features = ['release_date_day','release_date_month','release_date_weekday','release_date_weekofyear',
'release_date_year']
feature_text_length = ['title_count','tagline_count','original_title_length']
categorical_features = ['spoken_languages_all','production_companies_all','production_countries_all','original_language',
'got_collection','genres_all','crew_all','cast_all','belongs_to_three_lang','Keywords_all']
spoken_language_features = ['spoken_languages_','spoken_languages_Deutsch','spoken_languages_English','spoken_languages_Español',
'spoken_languages_Français','spoken_languages_Italiano', 'spoken_languages_Pусский',
'spoken_languages_हिन्दी','spoken_languages_日本語','spoken_languages_普通话']
production_countries_features = ['production_countries_Australia','production_countries_Canada','production_countries_France',
'production_countries_Germany','production_countries_India','production_countries_Italy',
'production_countries_Japan','production_countries_Russia','production_countries_United Kingdom',
'production_countries_United States of America' ]
production_companies_features = ['production_companies_Columbia Pictures','production_companies_Columbia Pictures Corporation',
'production_companies_Metro-Goldwyn-Mayer(MGM)', 'production_companies_New Line Cinema',
'production_companies_Paramount Pictures', 'production_companies_Touchstone Pictures',
'production_companies_Twentieth Century Fox Film Corporation','production_companies_Universal Pictures',
'production_companies_Walt Disney Pictures', 'production_companies_Warner Bros.']
genres_features = ['genres_Action', 'genres_Adventure','genres_Comedy', 'genres_Crime', 'genres_Drama', 'genres_Family',
'genres_Horror', 'genres_Romance', 'genres_Science Fiction', 'genres_Thriller']
crew_features = ['crew_Avy Kaufman', 'crew_Deborah Aquila', 'crew_Francine Maisler','crew_James Newton Howard',
'crew_Jerry Goldsmith', 'crew_Luc Besson', 'crew_Mary Vernieu', 'crew_Robert Rodriguez','crew_Steven Spielberg',
'crew_Tricia Wood']
cast_features = ['cast_Bruce McGill', 'cast_Bruce Willis','cast_J.K.Simmons','cast_John Turturro','cast_Liam Neeson',
'cast_Morgan Freeman','cast_Robert De Niro','cast_Samuel L.Jackson', 'cast_Susan Sarandon','cast_Willem Dafoe']
keywords_features = ['Keywords_aftercreditsstinger', 'Keywords_based on novel', 'Keywords_biography',
'Keywords_duringcreditsstinger', 'Keywords_independent film', 'Keywords_murder',
'Keywords_revenge', 'Keywords_sport', 'Keywords_violence', 'Keywords_woman director']
aug_imp.sort_values('importance_score', ascending=False, inplace=True)
 list(aug_imp.feature.values[:20] )<save_to_csv>
numerical_data = numerical_features
numerical_data.append('revenue' )
sub_aug = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv')
 sub_aug['time_to_failure'] = test_preds_aug
sub_aug.to_csv('sub_aug_data.csv', index=False )<import_modules>
def select_model(X_train, X_val, y_train, y_val):
best_models = {}
models = [
{
'name': 'LinearRegression',
'estimator': LinearRegression() ,
'hyperparameters': {},
},
{
'name': 'GradientBoostingRegressor',
'estimator': GradientBoostingRegressor() ,
'hyperparameters':{
'n_estimators': range(100, 200, 10),
'criterion': ['friedman_mse'],
'max_depth': [3, 5, 7, 9],
'max_features': ['log2', 'sqrt'],
'min_samples_leaf': [1, 2, 4],
'min_samples_split': [3, 5, 7]
}
},
{
'name': 'XGBoost',
'estimator': xgb.XGBRegressor() ,
'hyperparameters':{
'booster': ['gbtree', 'gblinear', 'dart'],
'max_depth': range(5, 50, 5),
'n_estimators': [200],
'nthread': [4],
'min_child_weight': range(1, 8, 2),
'learning_rate': [.05,.1,.15],
}
},
{
'name': 'Light GBM',
'estimator': lgb.LGBMRegressor() ,
'hyperparameters':{
'max_depth': range(20, 85, 15),
'learning_rate': [.01,.05,.1],
'num_leaves': [300, 600, 900, 1200],
'n_estimators': [200]
}
}
]
for model in tqdm(models):
print('
', '-'*25, '
', model['name'])
 start = time.perf_counter()
grid = GridSearchCV(model['estimator'], param_grid=model['hyperparameters'], cv=5, scoring = "neg_mean_squared_error", verbose=False, n_jobs=-1)
 grid.fit(X_train, y_train)
 best_models[model['name']] = {'score': grid.best_score_, 'params': grid.best_params_}
mse_val = mean_squared_error(y_val, grid.predict(X_val))
mse_train = mean_squared_error(y_train, grid.predict(X_train))
print("RMSLE train:{}".format(np.sqrt(mse_train)))
 print("RMSLE validation:{}".format(np.sqrt(mse_val)))
 print("best_params_:{}".format(grid.best_params_))
run = time.perf_counter() - start
return best_models
from sklearn.preprocessing import MinMaxScaler<normalization>
def get_best_parameters(train,features_list):
X_train = train[features_list]
y_train = np.log1p(train["revenue"] ).values
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.20, random_state=1)
 models = select_model(X_train, X_val, y_train, y_val)
 return models
initial_imp.loc[initial_imp.importance_score<-100, 'importance_score'] = -50
aug_imp.loc[aug_imp.importance_score<-100,'importance_score'] = -100
scaler = MinMaxScaler()
initial_imp.iloc[:, 1:] = scaler.fit_transform(initial_imp.iloc[:, 1:])
 scaler = MinMaxScaler()
aug_imp.iloc[:, 1:] = scaler.fit_transform(aug_imp.iloc[:, 1:] )<merge>
class Models(object):
def __init__(self, x_train, x_validation, y_train, y_validation):
self.x_train = [x_train.iloc[i].tolist() for i in range(len(x_train)) ]
self.x_validation = [x_validation.iloc[i].tolist() for i in range(len(x_validation)) ]
self.y_train = y_train.tolist()
self.y_validation = y_validation.tolist()
@staticmethod
def print_info(cross_val_scores, mse_train,mse_val):
print("Cross Validation Scores: ", cross_val_scores)
 print("RMSLE train:{}".format(np.sqrt(mse_train)))
 print("RMSLE validation:{}".format(np.sqrt(mse_val)))
 def GBR(self, x_train, x_validation, y_train, y_validation):
gbr = GradientBoostingRegressor(n_estimators=120, learning_rate=0.08,max_features='sqrt',criterion='friedman_mse',
min_samples_leaf=1,min_samples_split=3, max_depth=7, random_state=seed)
 gbr.fit(self.x_train, self.y_train)
 kfold = KFold(n_splits=10, shuffle=True, random_state=seed)
 cross_val_scores = cross_val_score(gbr, self.x_train, self.y_train, cv=kfold)
 mse_val = mean_squared_error(self.y_validation, gbr.predict(self.x_validation))
mse_train = mean_squared_error(self.y_train, gbr.predict(self.x_train))
print('
Gradient Boosting Regressor')
 self.print_info(cross_val_scores, mse_train,mse_val)
 return cross_val_scores, mse_val, gbr
def lgbm(self, x_train, x_validation, y_train, y_validation):
lgbm =lgb.LGBMRegressor(n_estimators=10000,objective="regression", metric="rmse",num_leaves=20,
min_child_samples=100,learning_rate=0.01, bagging_fraction=0.8,feature_fraction=0.8,
bagging_frequency=1,importance_type='gain', bagging_seed=seed,subsample=.9,
colsample_bytree=.9,use_best_model=True)
 lgbm.fit(x_train, y_train,eval_set=(x_validation, y_validation),verbose=False)
 kfold = KFold(n_splits=10, shuffle=True, random_state=seed)
 cross_val_scores = cross_val_score(lgbm, self.x_train, self.y_train, cv=kfold)
 mse_val = mean_squared_error(self.y_validation, lgbm.predict(self.x_validation))
mse_train = mean_squared_error(self.y_train, lgbm.predict(self.x_train))
print('
LGBM Regressor')
 self.print_info(cross_val_scores, mse_train,mse_val)
 return cross_val_scores, mse_val, lgbm
def xgBoost(self, x_train, x_validation, y_train, y_validation):
params = {'objective': 'reg:linear','eta': 0.01,'max_depth': 6,'subsample': 0.6,'colsample_bytree': 0.7,
'eval_metric': 'rmse', 'seed': seed,'silent': True,}
record = dict()
xg = xgb.train(params, xgb.DMatrix(x_train, y_train), 100000, [(xgb.DMatrix(x_train, y_train), 'train'), 
(xgb.DMatrix(x_validation, y_validation), 'valid')]
, verbose_eval=False, early_stopping_rounds=500, callbacks = [xgb.callback.record_evaluation(record)])
 best_idx = np.argmin(np.array(record['valid']['rmse']))
cross_val_scores= 0
mse_val = mean_squared_error(self.y_validation, xg.predict(xgb.DMatrix(x_validation), ntree_limit=xg.best_ntree_limit))
mse_train = mean_squared_error(self.y_train, xg.predict(xgb.DMatrix(x_train), ntree_limit=xg.best_ntree_limit))
print('
XgBoost Regressor')
 self.print_info(cross_val_scores, mse_train,mse_val)
 return cross_val_scores, mse_val, xg
aug_cols = ['feature'] + [x + '_all' for x in aug_imp.columns if x != 'feature']
aug_imp.columns = aug_cols
feature_df = initial_imp.merge(aug_imp, on='feature', how='inner')
 feature_df['score_change'] = feature_df['importance_score_all'] - feature_df['importance_score']
feature_df.sort_values('score_change', ascending=False, inplace=True)
 feature_df.head(10 )<split>
def evaluate_models(train, test,features_list):
X_train = train[features_list]
y_train = np.log1p(train["revenue"] ).values
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.20, random_state=1)
 methods = [x for x, y in Models.__dict__.items() if type(y)== FunctionType]
methods.remove('__init__')
 cross_scores_list, mse_list = [], []
models = {}
for model in methods:
reg = Models(X_train, X_val, y_train, y_val)
 cross_val_scores, mse, return_model = getattr(reg, model )(X_train, X_val, y_train, y_val)
 cross_scores_list.append(cross_val_scores)
 models[model] = return_model
mse_list.append(mse)
 return models
MIN_SCORE = np.percentile(feature_df.score_change.values.flatten() , 20)
 best_features = feature_df.loc[feature_df.score_change >= MIN_SCORE, :].feature.values.flatten()
train = train[best_features]
test = test[best_features]
train_select = train[index_augment]<train_on_grid>
corr_features_models = evaluate_models(train, test,corr_selected_features )
params_final = quick_hyperopt(train, y, 'lgbm', 150 )<define_variables>
all_features_models = evaluate_models(train, test,list(columns_to_keep))
<save_to_csv><EOS> 
def calculate_test_results(models,test,model_names,features_list):
X_test = test[features_list]
pred = np.empty(shape=len(X_test)).tolist()
for model in model_names:
mod = models[model]
if(model=='xgBoost'):
pred = pred + np.expm1(mod.predict(xgb.DMatrix(X_test), ntree_limit=mod.best_ntree_limit))
else:
pred = pred + np.expm1(mod.predict(X_test))
return pred