Kaggle competition: TMDB Box Office Prediction
<SOS> metric: RMSLE Kaggle data source: tmdb-box-office-prediction<train_on_grid>
pd.set_option('max_columns', None)
 %matplotlib inline
plt.style.use('ggplot')
 stop = set(stopwords.words('english'))
py.init_notebook_mode(connected=True)
%%time
ENet_model = ElasticNet(alpha=0.0005, l1_ratio=0.9, random_state=3,max_iter=5000)
 ".format(score.mean() , score.std()))
ENet_model<choose_model_class>
train = pd.read_csv('.. /input/tmdb-box-office-prediction/train.csv')
 test = pd.read_csv('.. /input/tmdb-box-office-prediction/test.csv')
 dict_columns = ['belongs_to_collection', 'genres', 'production_companies',
'production_countries', 'spoken_languages', 'Keywords', 'cast', 'crew']
def text_to_dict(df):
for column in dict_columns:
df[column] = df[column].apply(lambda x: {} if pd.isna(x)else ast.literal_eval(x))
return df
train = text_to_dict(train)
 test = text_to_dict(test )
%%time
lasso_model = Lasso(alpha =0.0005, random_state=1)
 ".format(score.mean() , score.std()))
lasso_model<train_model>
for i, e in enumerate(train['belongs_to_collection'][:5]):
print(i, e )
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )<choose_model_class>
train['belongs_to_collection'].apply(lambda x: len(x)if x != {} else 0 ).value_counts()
%%time
averaged_models = AveragingModels(models =(rf_model,cat_model))
".format(score_rf_cat.mean() , score_rf_cat.std()))
".format(score_cat_enet.mean() , score_cat_enet.std()))
".format(score_cat_lasso.mean() , score_cat_lasso.std()))<compute_train_metric>
train['collection_name'] = train['belongs_to_collection'].apply(lambda x: x[0]['name'] if x != {} else 0)
 train['has_collection'] = train['belongs_to_collection'].apply(lambda x: len(x)if x != {} else 0)
 test['collection_name'] = test['belongs_to_collection'].apply(lambda x: x[0]['name'] if x != {} else 0)
 test['has_collection'] = test['belongs_to_collection'].apply(lambda x: len(x)if x != {} else 0)
 train = train.drop(['belongs_to_collection'], axis=1)
 test = test.drop(['belongs_to_collection'], axis=1 )
averaged_models.fit(train_X_less.values, train_y)
 averaged_train_predict = averaged_models.predict(train_X_less.values)
 print(mean_absolute_error(train_y, averaged_train_predict))<predict_on_test>
for i, e in enumerate(train['genres'][:5]):
print(i, e )
averaged_prediction = np.zeros(len(test_X_less))
averaged_prediction += averaged_models.predict(test_X_less.values)
 averaged_prediction<load_from_csv>
print('Number of genres in films')
 train['genres'].apply(lambda x: len(x)if x != {} else 0 ).value_counts()
submission = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv', index_col='seg_id')
 submission.head()<save_to_csv>
list_of_genres = list(train['genres'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values )
submission.time_to_failure = averaged_prediction
submission.to_csv('submission_average_167_features.csv',index=True )<set_options>
train['num_genres'] = train['genres'].apply(lambda x: len(x)if x != {} else 0)
 train['all_genres'] = train['genres'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_genres = [m[0] for m in Counter([i for j in list_of_genres for i in j] ).most_common(15)]
for g in top_genres:
train['genre_' + g] = train['all_genres'].apply(lambda x: 1 if g in x else 0)
 test['num_genres'] = test['genres'].apply(lambda x: len(x)if x != {} else 0)
 test['all_genres'] = test['genres'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_genres:
test['genre_' + g] = test['all_genres'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['genres'], axis=1)
 test = test.drop(['genres'], axis=1 )
warnings.filterwarnings("ignore")
 PandasDataFrame = TypeVar('pandas.core.frame.DataFrame')
<load_from_csv>
for i, e in enumerate(train['production_companies'][:5]):
print(i, e )
train_X = pd.read_csv('.. /input/mlip-data/train_features.csv')
 test_X = pd.read_csv('.. /input/mlip-data/test_features.csv')
 train_y = pd.read_csv('.. /input/mlip-data/y.csv' )<load_from_csv>
print('Number of production companies in films')
 train['production_companies'].apply(lambda x: len(x)if x != {} else 0 ).value_counts()
submission = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv', index_col='seg_id')
 submission.shape<choose_model_class>
train[train['production_companies'].apply(lambda x: len(x)if x != {} else 0)> 11]
mse_scoring = make_scorer(mean_squared_error, greater_is_better=False)
 mae_scoring = make_scorer(mean_absolute_error, greater_is_better=False)
 k_fold = KFold(n_splits=7, shuffle=True, random_state=13 )<train_model>
img = Image.open(urlopen("https://image.tmdb.org/t/p/w600_and_h900_bestv2/y31QB9kn3XSudA15tV7UWQ9XLuW.jpg"))
img
def run_lgb(
 X_tr: PandasDataFrame,
X_val: PandasDataFrame,
y_tr: PandasDataFrame,
y_val: PandasDataFrame,
test_data: PandasDataFrame,
params: Dict 
):

 early_stop = 200
num_rounds = 10000
verbose_eval=1000
d_train = lgb.Dataset(X_tr, label=y_tr)
 d_valid = lgb.Dataset(X_val, label=y_val)
 watchlist = [d_train, d_valid]
model = lgb.train(
 params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop 
)
 val_pred = model.predict(X_val, num_iteration=model.best_iteration)
 prediction = model.predict(test_data, num_iteration=model.best_iteration)
 return val_pred, prediction, model.feature_importance(importance_type="split")
 def run_cv_model(
 train_data,
train_target,
test_data,
model_fn, params,
scoring,
label='model',
feature_imp=False,
n_folds = 5 
):
oof_val = np.zeros(len(train_data))
predictions = np.zeros(len(test_data))
oof_predict = np.zeros(( n_folds, test_data.shape[0]))
scores = []
feature_importance_df = pd.DataFrame()
folds = KFold(n_splits=n_folds, shuffle=True, random_state=42)
 train_columns = train_data.columns.values
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_data, train_target.values)) :
strLog = "fold {} for {}".format(fold_, label)
 print(strLog)
 X_tr, X_val = train_data.iloc[trn_idx], train_data.iloc[val_idx]
y_tr, y_val = train_target.iloc[trn_idx], train_target.iloc[val_idx]
val_pred, prediction, feature_importances = model_fn(
 X_tr, X_val,
y_tr, y_val,
test_data,
params 
)
 score = scoring(y_val, val_pred)
 scores.append(score)
 oof_val[val_idx] = val_pred
if feature_imp == True:
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = train_columns
fold_importance_df["importance"] = feature_importances[:len(train_columns)]
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 oof_predict[fold_] = prediction
predictions += prediction/folds.n_splits
print('CV score: {0:.4f}, std: {1:.4f}.
 '.format(np.mean(score), np.std(score)))
 print('CV mean score: {0:.4f}, std: {1:.4f}.
 '.format(np.mean(scores), np.std(scores)))
 return oof_val, oof_predict, predictions, feature_importance_df, scores<choose_model_class>
list_of_companies = list(train['production_companies'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values )
def LGB_bayesian(**lgb_bayes_params):
lgb_bayes_params['num_leaves'] = int(lgb_bayes_params['num_leaves'])
 lgb_bayes_params['max_depth'] = int(lgb_bayes_params['max_depth'])
 lgb_bayes_params['min_child_samples'] = int(lgb_bayes_params['min_child_samples'])
 lgb_bayes_params['max_bin'] = int(lgb_bayes_params['max_bin'])
 lgb_bayes_params['subsample_freq'] = int(lgb_bayes_params['subsample_freq'])
 lgb_bayes_params['min_child_weight'] = int(lgb_bayes_params['min_child_weight'])
 lgb_bayes_params['subsample_for_bin'] = int(lgb_bayes_params['subsample_for_bin'])
 lgb_bayes_params['n_estimators'] = int(lgb_bayes_params['n_estimators'])
 fix_lgb_params = {
'objective':'regression',
"boosting": "gbdt",
"metric": 'mae',
}
lgb_params = {**fix_lgb_params, **lgb_bayes_params}
oof_val, oof_predict, predictions, feature_importance_df, scores = run_cv_model(
 train_data=train_X,
train_target=train_y,
test_data=test_X,
model_fn=run_lgb,
params=lgb_params,
scoring=mean_absolute_error,
label="lgb",
feature_imp=True,
n_folds=8 
)
 return -np.mean(scores )<define_search_space>
train['num_companies'] = train['production_companies'].apply(lambda x: len(x)if x != {} else 0)
 train['all_production_companies'] = train['production_companies'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_companies = [m[0] for m in Counter([i for j in list_of_companies for i in j] ).most_common(30)]
for g in top_companies:
train['production_company_' + g] = train['all_production_companies'].apply(lambda x: 1 if g in x else 0)
 test['num_companies'] = test['production_companies'].apply(lambda x: len(x)if x != {} else 0)
 test['all_production_companies'] = test['production_companies'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_companies:
test['production_company_' + g] = test['all_production_companies'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['production_companies', 'all_production_companies'], axis=1)
 test = test.drop(['production_companies', 'all_production_companies'], axis=1 )
search_spaces = {
'learning_rate':(0.01,.5),
'num_leaves':(2, 350),
'max_depth':(50, 500),
'min_child_samples':(20, 150),
'max_bin':(30000, 100000),
'subsample':(0.01, 1.0),
'subsample_freq':(2, 20),
'colsample_bytree':(0.01, 1.0),
'min_child_weight':(0, 10),
'subsample_for_bin':(100000, 800000),
'reg_lambda':(1e-9, 1000),
'reg_alpha':(1e-9, 1.0),
'scale_pos_weight':(1e-6, 1000),
'n_estimators':(1000, 10000),
'min_gain_to_split':(0.0, 1.0),
}
lgb_bo = BayesianOptimization(LGB_bayesian,search_spaces, random_state=7)
<init_hyperparams>
for i, e in enumerate(train['production_countries'][:5]):
print(i, e )
lgb_bayes_params = lgb_bo.max['params']
lgb_bayes_params['num_leaves'] = int(lgb_bayes_params['num_leaves'])
 lgb_bayes_params['max_depth'] = int(lgb_bayes_params['max_depth'])
 lgb_bayes_params['min_child_samples'] = int(lgb_bayes_params['min_child_samples'])
 lgb_bayes_params['max_bin'] = int(lgb_bayes_params['max_bin'])
 lgb_bayes_params['subsample_freq'] = int(lgb_bayes_params['subsample_freq'])
 lgb_bayes_params['min_child_weight'] = int(lgb_bayes_params['min_child_weight'])
 lgb_bayes_params['subsample_for_bin'] = int(lgb_bayes_params['subsample_for_bin'])
 lgb_bayes_params['n_estimators'] = int(lgb_bayes_params['n_estimators'] )<init_hyperparams>
print('Number of production countries in films')
 train['production_countries'].apply(lambda x: len(x)if x != {} else 0 ).value_counts()
lgb_bayes_params = {'colsample_bytree': 0.07731235427729943,
'learning_rate': 0.021390697326123965,
'max_bin': 52576,
'max_depth': 339,
'min_child_samples': 119,
'min_child_weight': 6,
'min_gain_to_split': 0.4223894968003159,
'n_estimators': 1560,
'num_leaves': 25,
'reg_alpha': 0.7079338662764831,
'reg_lambda': 368.73133802650744,
'scale_pos_weight': 363.3220921999365,
'subsample': 0.9910157826344941,
'subsample_for_bin': 693174,
'subsample_freq': 19}<import_modules>
list_of_countries = list(train['production_countries'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_countries for i in j] ).most_common(25 )
class FeatureSelector() :

 def __init__(self, data, labels=None):
self.data = data
self.labels = labels
if labels is None:
print('No labels provided.Feature importance based methods are not available.')
 self.base_features = list(data.columns)
 self.one_hot_features = None
self.record_missing = None
self.record_single_unique = None
self.record_collinear = None
self.record_zero_importance = None
self.record_low_importance = None
self.missing_stats = None
self.unique_stats = None
self.corr_matrix = None
self.feature_importances = None
self.ops = {}
self.one_hot_correlated = False
def identify_missing(self, missing_threshold):

 self.missing_threshold = missing_threshold
missing_series = self.data.isnull().sum() / self.data.shape[0]
self.missing_stats = pd.DataFrame(missing_series ).rename(columns = {'index': 'feature', 0: 'missing_fraction'})
 self.missing_stats = self.missing_stats.sort_values('missing_fraction', ascending = False)
 record_missing = pd.DataFrame(missing_series[missing_series > missing_threshold] ).reset_index().rename(columns =
{'index': 'feature',
0: 'missing_fraction'})
 to_drop = list(record_missing['feature'])
 self.record_missing = record_missing
self.ops['missing'] = to_drop
print('%d features with greater than %0.2f missing values.
 ' %(len(self.ops['missing']), self.missing_threshold))
def identify_single_unique(self):

 unique_counts = self.data.nunique()
self.unique_stats = pd.DataFrame(unique_counts ).rename(columns = {'index': 'feature', 0: 'nunique'})
 self.unique_stats = self.unique_stats.sort_values('nunique', ascending = True)
 record_single_unique = pd.DataFrame(unique_counts[unique_counts == 1] ).reset_index().rename(columns = {'index': 'feature',
0: 'nunique'})
 to_drop = list(record_single_unique['feature'])
 self.record_single_unique = record_single_unique
self.ops['single_unique'] = to_drop
print('%d features with a single unique value.
 ' % len(self.ops['single_unique']))
def identify_collinear(self, correlation_threshold, one_hot=False):

 self.correlation_threshold = correlation_threshold
self.one_hot_correlated = one_hot
if one_hot:
features = pd.get_dummies(self.data)
 self.one_hot_features = [column for column in features.columns if column not in self.base_features]
self.data_all = pd.concat([features[self.one_hot_features], self.data], axis = 1)
 corr_matrix = pd.get_dummies(features ).corr()
else:
corr_matrix = self.data.corr()
self.corr_matrix = corr_matrix
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k = 1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(upper[column].abs() > correlation_threshold)]
record_collinear = pd.DataFrame(columns = ['drop_feature', 'corr_feature', 'corr_value'])
 for column in to_drop:
corr_features = list(upper.index[upper[column].abs() > correlation_threshold])
 corr_values = list(upper[column][upper[column].abs() > correlation_threshold])
 drop_features = [column for _ in range(len(corr_features)) ]
temp_df = pd.DataFrame.from_dict({'drop_feature': drop_features,
'corr_feature': corr_features,
'corr_value': corr_values})
 record_collinear = record_collinear.append(temp_df, ignore_index = True)
 self.record_collinear = record_collinear
self.ops['collinear'] = to_drop
print('%d features with a correlation magnitude greater than %0.2f.
 ' %(len(self.ops['collinear']), self.correlation_threshold))
def identify_zero_importance(self, feature_importances):

 if self.labels is None:
raise ValueError("No training labels provided.")
 features = pd.get_dummies(self.data)
 self.one_hot_features = [column for column in features.columns if column not in self.base_features]
self.data_all = pd.concat([features[self.one_hot_features], self.data], axis = 1)
 feature_names = list(features.columns)
 features = np.array(features)
 labels = np.array(self.labels ).reshape(( -1,))
feature_importances = feature_importances.sort_values('importance', ascending = False ).reset_index(drop = True)
 feature_importances['normalized_importance'] = feature_importances['importance'] / feature_importances['importance'].sum()
feature_importances['cumulative_importance'] = np.cumsum(feature_importances['normalized_importance'])
 record_zero_importance = feature_importances[feature_importances['importance'] == 0.0]
to_drop = list(record_zero_importance['feature'])
 self.feature_importances = feature_importances
self.record_zero_importance = record_zero_importance
self.ops['zero_importance'] = to_drop
print('
%d features with zero importance after one-hot encoding.
 ' % len(self.ops['zero_importance']))
def identify_low_importance(self, cumulative_importance):

 self.cumulative_importance = cumulative_importance
if self.feature_importances is None:
raise NotImplementedError()
self.feature_importances = self.feature_importances.sort_values('cumulative_importance')
 record_low_importance = self.feature_importances[self.feature_importances['cumulative_importance'] > cumulative_importance]
to_drop = list(record_low_importance['feature'])
 self.record_low_importance = record_low_importance
self.ops['low_importance'] = to_drop
print('%d features required for cumulative importance of %0.2f after one hot encoding.' %(len(self.feature_importances)-
len(self.record_low_importance), self.cumulative_importance))
print('%d features do not contribute to cumulative importance of %0.2f.
 ' %(len(self.ops['low_importance']),
self.cumulative_importance))
def identify_all(self, selection_params):

 for param in ['missing_threshold', 'correlation_threshold', 'eval_metric', 'task', 'cumulative_importance']:
if param not in selection_params.keys() :
raise ValueError('%s is a required parameter for this method.' % param)
 self.identify_missing(selection_params['missing_threshold'])
 self.identify_single_unique()
self.identify_collinear(selection_params['correlation_threshold'])
 self.identify_zero_importance(task = selection_params['task'], eval_metric = selection_params['eval_metric'])
 self.identify_low_importance(selection_params['cumulative_importance'])
 self.all_identified = set(list(chain(*list(self.ops.values()))))
self.n_identified = len(self.all_identified)
 print('%d total features out of %d identified for removal after one-hot encoding.
 ' %(self.n_identified,
self.data_all.shape[1]))
def check_removal(self, keep_one_hot=True):

 self.all_identified = set(list(chain(*list(self.ops.values()))))
print('Total of %d features identified for removal' % len(self.all_identified))
if not keep_one_hot:
if self.one_hot_features is None:
print('Data has not been one-hot encoded')
 else:
one_hot_to_remove = [x for x in self.one_hot_features if x not in self.all_identified]
print('%d additional one-hot features can be removed' % len(one_hot_to_remove))
return list(self.all_identified)
 def remove(self, methods, keep_one_hot = True):

 features_to_drop = []
if methods == 'all':
data = self.data_all
print('{} methods have been run
'.format(list(self.ops.keys())))
 features_to_drop = set(list(chain(*list(self.ops.values()))))
else:
if 'zero_importance' in methods or 'low_importance' in methods or self.one_hot_correlated:
data = self.data_all
else:
data = self.data
for method in methods:
if method not in self.ops.keys() :
raise NotImplementedError('%s method has not been run' % method)
 else:
features_to_drop.append(self.ops[method])
 features_to_drop = set(list(chain(*features_to_drop)))
 features_to_drop = list(features_to_drop)
 if not keep_one_hot:
if self.one_hot_features is None:
print('Data has not been one-hot encoded')
 else:
features_to_drop = list(set(features_to_drop)| set(self.one_hot_features))
data = data.drop(columns = features_to_drop)
 self.removed_features = features_to_drop
if not keep_one_hot:
	print('Removed %d features including one-hot features.' % len(features_to_drop))
else:
	print('Removed %d features.' % len(features_to_drop))
return data
def plot_missing(self):

 if self.record_missing is None:
raise NotImplementedError("Missing values have not been calculated.Run `identify_missing`")
 self.reset_plot()
plt.style.use('seaborn-white')
 plt.figure(figsize =(7, 5))
plt.hist(self.missing_stats['missing_fraction'], bins = np.linspace(0, 1, 11), edgecolor = 'k', color = 'red', linewidth = 1.5)
 plt.xticks(np.linspace(0, 1, 11)) ;
plt.xlabel('Missing Fraction', size = 14); plt.ylabel('Count of Features', size = 14);
plt.title("Fraction of Missing Values Histogram", size = 16);
def plot_unique(self):

 if self.record_single_unique is None:
raise NotImplementedError('Unique values have not been calculated.Run `identify_single_unique`')
 self.reset_plot()
self.unique_stats.plot.hist(edgecolor = 'k', figsize =(7, 5))
plt.ylabel('Frequency', size = 14); plt.xlabel('Unique Values', size = 14);
plt.title('Number of Unique Values Histogram', size = 16);
def plot_collinear(self, plot_all = False):

 if self.record_collinear is None:
raise NotImplementedError('Collinear features have not been idenfitied.Run `identify_collinear`.')
 if plot_all:
	corr_matrix_plot = self.corr_matrix
	title = 'All Correlations'
else:
	
	
	 corr_matrix_plot = self.corr_matrix.loc[list(set(self.record_collinear['corr_feature'])) ,
	 list(set(self.record_collinear['drop_feature'])) ]
	 title = "Correlations Above Threshold"
f, ax = plt.subplots(figsize=(10, 8))
cmap = sns.diverging_palette(220, 10, as_cmap=True)
 sns.heatmap(corr_matrix_plot, cmap=cmap, center=0,
linewidths=.25, cbar_kws={"shrink": 0.6})
 ax.set_yticks([x + 0.5 for x in list(range(corr_matrix_plot.shape[0])) ])
 ax.set_yticklabels(list(corr_matrix_plot.index), size = int(160 / corr_matrix_plot.shape[0])) ;
ax.set_xticks([x + 0.5 for x in list(range(corr_matrix_plot.shape[1])) ])
 ax.set_xticklabels(list(corr_matrix_plot.columns), size = int(160 / corr_matrix_plot.shape[1])) ;
plt.title(title, size = 14)
 def plot_feature_importances(self, plot_n = 15, threshold = None):

 if self.record_zero_importance is None:
raise NotImplementedError('Feature importances have not been determined.Run `idenfity_zero_importance`')
 if plot_n > self.feature_importances.shape[0]:
plot_n = self.feature_importances.shape[0] - 1
self.reset_plot()
plt.figure(figsize =(10, 6))
ax = plt.subplot()
ax.barh(list(reversed(list(self.feature_importances.index[:plot_n]))),
self.feature_importances['normalized_importance'][:plot_n],
align = 'center', edgecolor = 'k')
 ax.set_yticks(list(reversed(list(self.feature_importances.index[:plot_n]))))
ax.set_yticklabels(self.feature_importances['feature'][:plot_n], size = 12)
 plt.xlabel('Normalized Importance', size = 16); plt.title('Feature Importances', size = 18)
 plt.show()
plt.figure(figsize =(6, 4))
plt.plot(list(range(1, len(self.feature_importances)+ 1)) , self.feature_importances['cumulative_importance'], 'r-')
 plt.xlabel('Number of Features', size = 14); plt.ylabel('Cumulative Importance', size = 14);
plt.title('Cumulative Feature Importance', size = 16);
if threshold:
importance_index = np.min(np.where(self.feature_importances['cumulative_importance'] > threshold))
plt.vlines(x = importance_index + 1, ymin = 0, ymax = 1, linestyles='--', colors = 'blue')
 plt.show() ;
print('%d features required for %0.2f of cumulative importance' %(importance_index + 1, threshold))
def reset_plot(self):
plt.rcParams = plt.rcParamsDefault<compute_train_metric>
train['num_countries'] = train['production_countries'].apply(lambda x: len(x)if x != {} else 0)
 train['all_countries'] = train['production_countries'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_countries = [m[0] for m in Counter([i for j in list_of_countries for i in j] ).most_common(25)]
for g in top_countries:
train['production_country_' + g] = train['all_countries'].apply(lambda x: 1 if g in x else 0)
 test['num_countries'] = test['production_countries'].apply(lambda x: len(x)if x != {} else 0)
 test['all_countries'] = test['production_countries'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_countries:
test['production_country_' + g] = test['all_countries'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['production_countries', 'all_countries'], axis=1)
 test = test.drop(['production_countries', 'all_countries'], axis=1 )
fix_lgb_params = {
'objective':'regression',
"boosting": "gbdt",
"metric": 'mae',
}
lgb_params = {**fix_lgb_params, **lgb_bayes_params}
oof_val, oof_predict, predictions, feature_importance_df, scores = run_cv_model(
 train_data=train_X,
train_target=train_y,
test_data=test_X,
model_fn=run_lgb,
params=lgb_params,
scoring=mean_absolute_error,
label="lgb",
feature_imp=True,
n_folds=8
)<save_to_csv>
for i, e in enumerate(train['spoken_languages'][:5]):
print(i, e )
submission.to_csv('submission.csv',index=True)
 fs = FeatureSelector(data = train_X, labels = train_y)
 feature_importance = feature_importance_df[["feature", "importance"]].groupby("feature" ).mean()
feature_importance['feature'] = feature_importance.index<define_variables>
print('Number of spoken languages in films')
 train['spoken_languages'].apply(lambda x: len(x)if x != {} else 0 ).value_counts()
one_hot_features = fs.one_hot_features
base_features = fs.base_features
print('There are %d original features' % len(base_features))
print('There are %d one-hot features' % len(one_hot_features))<drop_column>
list_of_languages = list(train['spoken_languages'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_languages for i in j] ).most_common(15 )
all_to_remove = fs.check_removal()<drop_column>
train['num_languages'] = train['spoken_languages'].apply(lambda x: len(x)if x != {} else 0)
 train['all_languages'] = train['spoken_languages'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_languages = [m[0] for m in Counter([i for j in list_of_languages for i in j] ).most_common(30)]
for g in top_languages:
train['language_' + g] = train['all_languages'].apply(lambda x: 1 if g in x else 0)
 test['num_languages'] = test['spoken_languages'].apply(lambda x: len(x)if x != {} else 0)
 test['all_languages'] = test['spoken_languages'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_languages:
test['language_' + g] = test['all_languages'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['spoken_languages', 'all_languages'], axis=1)
 test = test.drop(['spoken_languages', 'all_languages'], axis=1 )
train_removed_all = fs.remove(methods = 'all', keep_one_hot=False )<choose_model_class>
for i, e in enumerate(train['Keywords'][:5]):
print(i, e )
def LGB_bayesian_sec(**lgb_bayes_params):
lgb_bayes_params['num_leaves'] = int(lgb_bayes_params['num_leaves'])
 lgb_bayes_params['max_depth'] = int(lgb_bayes_params['max_depth'])
 lgb_bayes_params['min_child_samples'] = int(lgb_bayes_params['min_child_samples'])
 lgb_bayes_params['max_bin'] = int(lgb_bayes_params['max_bin'])
 lgb_bayes_params['subsample_freq'] = int(lgb_bayes_params['subsample_freq'])
 lgb_bayes_params['min_child_weight'] = int(lgb_bayes_params['min_child_weight'])
 lgb_bayes_params['subsample_for_bin'] = int(lgb_bayes_params['subsample_for_bin'])
 lgb_bayes_params['n_estimators'] = int(lgb_bayes_params['n_estimators'])
 fix_lgb_params = {
'objective':'regression',
"boosting": "gbdt",
"metric": 'mae',
}
lgb_params = {**fix_lgb_params, **lgb_bayes_params}
oof_val, oof_predict, predictions, feature_importance_df, scores = run_cv_model(
 train_data=train_df,
train_target=train_y,
test_data=test_df,
model_fn=run_lgb,
params=lgb_params,
scoring=mean_absolute_error,
label="lgb",
feature_imp=True,
n_folds=8 
)
 return -np.mean(scores )<choose_model_class>
print('Number of Keywords in films')
 train['Keywords'].apply(lambda x: len(x)if x != {} else 0 ).value_counts().head(10 )
lgb_bo = BayesianOptimization(LGB_bayesian_sec,search_spaces, random_state=7)
<set_options>
list_of_keywords = list(train['Keywords'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 plt.figure(figsize =(16, 12))
text = ' '.join(['_'.join(i.split(' ')) for j in list_of_keywords for i in j])
 wordcloud = WordCloud(max_font_size=None, background_color='black', collocations=False,
width=1200, height=1000 ).generate(text)
 plt.imshow(wordcloud)
 plt.title('Top keywords')
 plt.axis("off")
 plt.show()
with warnings.catch_warnings() :
warnings.filterwarnings('ignore')
 lgb_bo.maximize(init_points=50, n_iter=50, acq='ucb' )<init_hyperparams>
train['num_Keywords'] = train['Keywords'].apply(lambda x: len(x)if x != {} else 0)
 train['all_Keywords'] = train['Keywords'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_keywords = [m[0] for m in Counter([i for j in list_of_keywords for i in j] ).most_common(30)]
for g in top_keywords:
train['keyword_' + g] = train['all_Keywords'].apply(lambda x: 1 if g in x else 0)
 test['num_Keywords'] = test['Keywords'].apply(lambda x: len(x)if x != {} else 0)
 test['all_Keywords'] = test['Keywords'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_keywords:
test['keyword_' + g] = test['all_Keywords'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['Keywords', 'all_Keywords'], axis=1)
 test = test.drop(['Keywords', 'all_Keywords'], axis=1 )
lgb_bayes_params = lgb_bo.max['params']
lgb_bayes_params['num_leaves'] = int(lgb_bayes_params['num_leaves'])
 lgb_bayes_params['max_depth'] = int(lgb_bayes_params['max_depth'])
 lgb_bayes_params['min_child_samples'] = int(lgb_bayes_params['min_child_samples'])
 lgb_bayes_params['max_bin'] = int(lgb_bayes_params['max_bin'])
 lgb_bayes_params['subsample_freq'] = int(lgb_bayes_params['subsample_freq'])
 lgb_bayes_params['min_child_weight'] = int(lgb_bayes_params['min_child_weight'])
 lgb_bayes_params['subsample_for_bin'] = int(lgb_bayes_params['subsample_for_bin'])
 lgb_bayes_params['n_estimators'] = int(lgb_bayes_params['n_estimators'])
 lgb_params = {**fix_lgb_params, **lgb_bayes_params} 
<compute_train_metric>
for i, e in enumerate(train['cast'][:1]):
print(i, e )
oof_val, oof_predict, predictions, feature_importance_df, scores = run_cv_model(
 train_data=train_df,
train_target=train_y,
test_data=test_df,
model_fn=run_lgb,
params=lgb_params,
scoring=mean_absolute_error,
label="lgb",
feature_imp=True,
n_folds=8
)<save_to_csv>
print('Number of casted persons in films')
 train['cast'].apply(lambda x: len(x)if x != {} else 0 ).value_counts().head(10 )
submission.time_to_failure = predictions
submission.to_csv('clean_submission.csv',index=True )<install_modules>
list_of_cast_names = list(train['cast'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_cast_names for i in j] ).most_common(15 )
!apt-get remove swig
!apt-get install swig3.0 build-essential -y
!ln -s /usr/bin/swig3.0 /usr/bin/swig
!apt-get install build-essential
!pip install --upgrade setuptools
!pip install auto-sklearn
try:
except:
pass<install_modules>
list_of_cast_genders = list(train['cast'].apply(lambda x: [i['gender'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_cast_genders for i in j] ).most_common()
!apt-get remove swig
!apt-get install swig3.0 build-essential -y
!ln -s /usr/bin/swig3.0 /usr/bin/swig
!apt-get install build-essential
!pip install --upgrade setuptools
!pip install auto-sklearn 
<set_options>
list_of_cast_characters = list(train['cast'].apply(lambda x: [i['character'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_cast_characters for i in j] ).most_common(15 )
warnings.filterwarnings("ignore")
 warnings.simplefilter(action='ignore', category=FutureWarning)
<load_from_csv>
train['num_cast'] = train['cast'].apply(lambda x: len(x)if x != {} else 0)
 top_cast_names = [m[0] for m in Counter([i for j in list_of_cast_names for i in j] ).most_common(15)]
for g in top_cast_names:
train['cast_name_' + g] = train['cast'].apply(lambda x: 1 if g in str(x)else 0)
 train['genders_0_cast'] = train['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
train['genders_1_cast'] = train['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
train['genders_2_cast'] = train['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
top_cast_characters = [m[0] for m in Counter([i for j in list_of_cast_characters for i in j] ).most_common(15)]
for g in top_cast_characters:
train['cast_character_' + g] = train['cast'].apply(lambda x: 1 if g in str(x)else 0)
 test['num_cast'] = test['cast'].apply(lambda x: len(x)if x != {} else 0)
 for g in top_cast_names:
test['cast_name_' + g] = test['cast'].apply(lambda x: 1 if g in str(x)else 0)
 test['genders_0_cast'] = test['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
test['genders_1_cast'] = test['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
test['genders_2_cast'] = test['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
for g in top_cast_characters:
test['cast_character_' + g] = test['cast'].apply(lambda x: 1 if g in str(x)else 0)
 train = train.drop(['cast'], axis=1)
 test = test.drop(['cast'], axis=1 )
train_X_0 = pd.read_csv(".. /input/lanl-master-s-features-creating-0/train_X_features_865.csv")
 train_X_1 = pd.read_csv(".. /input/lanl-master-s-features-creating-1/train_X_features_865.csv")
 y_0 = pd.read_csv(".. /input/lanl-master-s-features-creating-0/train_y.csv", index_col=False, header=None)
 y_1 = pd.read_csv(".. /input/lanl-master-s-features-creating-1/train_y.csv", index_col=False, header=None )<concatenate>
print('Number of casted persons in films')
 train['crew'].apply(lambda x: len(x)if x != {} else 0 ).value_counts().head(10 )
train_X = pd.concat([train_X_0, train_X_1], axis=0)
 train_X = train_X.reset_index(drop=True)
 print(train_X.shape)
 train_X.head()<concatenate>
list_of_crew_names = list(train['crew'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_crew_names for i in j] ).most_common(15 )
train_y = pd.concat([y_0, y_1], axis=0)
 train_y = train_y.reset_index(drop=True)
 train_y.columns = ['time_to_failure']
train_y['time_to_failure'].shape<load_from_csv>
list_of_crew_jobs = list(train['crew'].apply(lambda x: [i['job'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_crew_jobs for i in j] ).most_common(15 )
test_X = pd.read_csv(".. /input/lanl-master-s-features-creating-0/test_X_features_10.csv")
 test_X.drop('seg_id',inplace=True,axis=1 )<normalization>
list_of_crew_genders = list(train['crew'].apply(lambda x: [i['gender'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_crew_genders for i in j] ).most_common(15 )
scaler = StandardScaler()
train_columns = train_X.columns
train_X[train_columns] = scaler.fit_transform(train_X[train_columns])
 test_X[train_columns] = scaler.transform(test_X[train_columns] )<train_model>
list_of_crew_departments = list(train['crew'].apply(lambda x: [i['department'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_crew_departments for i in j] ).most_common(14 )
cls = autosklearn.regression.AutoSklearnRegressor(ensemble_memory_limit=5000,ml_memory_limit=10000,time_left_for_this_task=10000,per_run_time_limit=1000,seed=123,resampling_strategy='cv',resampling_strategy_arguments={"folds": 5})
 cls.fit(train_X, train_y,metric=autosklearn.metrics.mean_absolute_error,dataset_name='LANL')
 cls.refit(train_X, train_y )<save_to_csv>
train['num_crew'] = train['crew'].apply(lambda x: len(x)if x != {} else 0)
 top_crew_names = [m[0] for m in Counter([i for j in list_of_crew_names for i in j] ).most_common(15)]
for g in top_crew_names:
train['crew_name_' + g] = train['crew'].apply(lambda x: 1 if g in str(x)else 0)
 train['genders_0_crew'] = train['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
train['genders_1_crew'] = train['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
train['genders_2_crew'] = train['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
top_cast_characters = [m[0] for m in Counter([i for j in list_of_cast_characters for i in j] ).most_common(15)]
for g in top_cast_characters:
train['crew_character_' + g] = train['crew'].apply(lambda x: 1 if g in str(x)else 0)
 top_crew_jobs = [m[0] for m in Counter([i for j in list_of_crew_jobs for i in j] ).most_common(15)]
for j in top_crew_jobs:
train['jobs_' + j] = train['crew'].apply(lambda x: sum([1 for i in x if i['job'] == j]))
top_crew_departments = [m[0] for m in Counter([i for j in list_of_crew_departments for i in j] ).most_common(15)]
for j in top_crew_departments:
train['departments_' + j] = train['crew'].apply(lambda x: sum([1 for i in x if i['department'] == j]))
test['num_crew'] = test['crew'].apply(lambda x: len(x)if x != {} else 0)
 for g in top_crew_names:
test['crew_name_' + g] = test['crew'].apply(lambda x: 1 if g in str(x)else 0)
 test['genders_0_crew'] = test['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
test['genders_1_crew'] = test['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
test['genders_2_crew'] = test['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
for g in top_cast_characters:
test['crew_character_' + g] = test['crew'].apply(lambda x: 1 if g in str(x)else 0)
 for j in top_crew_jobs:
test['jobs_' + j] = test['crew'].apply(lambda x: sum([1 for i in x if i['job'] == j]))
for j in top_crew_departments:
test['departments_' + j] = test['crew'].apply(lambda x: sum([1 for i in x if i['department'] == j]))
train = train.drop(['crew'], axis=1)
 test = test.drop(['crew'], axis=1 )
preds = cls.predict(test_X)
 sample_submission = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv')
 sample_submission['time_to_failure'] = preds.flatten()
sample_submission.to_csv('submission.csv', index=False )<set_options>
train['log_revenue'] = np.log1p(train['revenue'] )
%matplotlib inline
pd.options.display.precision = 15
warnings.filterwarnings("ignore")
<load_from_csv>
train['log_budget'] = np.log1p(train['budget'])
 test['log_budget'] = np.log1p(test['budget'] )
%%time
train = pd.read_csv('.. /input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32},nrows=6e8 )<count_unique_values>
train['homepage'].value_counts().head()
min = -100
max = 100
spread = 110
def get_counts(sequence):
counts = [0]*spread
unique_count = np.unique(sequence, return_counts=True)
 for i in range(0,len(unique_count[0])) :
val = unique_count[0][i]
count = unique_count[1][i]
r = count*val
if val <= min:
counts[0] += r
elif val >= max:
counts[-1] += r
else:
counts[int(val/2)+int(spread/2)] += r
return counts<create_dataframe>
train['has_homepage'] = 0
train.loc[train['homepage'].isnull() == False, 'has_homepage'] = 1
test['has_homepage'] = 0
test.loc[test['homepage'].isnull() == False, 'has_homepage'] = 1
interval = 75000
counts = [get_counts(train.acoustic_data.values[i:i+150000])for i in tqdm_notebook(range(0,len(train),interval)) ]
ttfs = [train.time_to_failure.values[i] for i in range(0,len(train),interval)]
del train
labels = ["D"+str(i)for i in range(0,len(counts[0])) ]
df = pd.DataFrame(counts, columns=labels)
 ttf_df = pd.DataFrame(ttfs, columns=["expected"])
 df = df.join(ttf_df )<load_from_csv>
vectorizer = TfidfVectorizer(
 sublinear_tf=True,
analyzer='word',
token_pattern=r'\w{1,}',
ngram_range=(1, 2),
min_df=5)
 overview_text = vectorizer.fit_transform(train['overview'].fillna(''))
linreg = LinearRegression()
linreg.fit(overview_text, train['log_revenue'])
 eli5.show_weights(linreg, vec=vectorizer, top=20, feature_filter=lambda x: x != '<BIAS>' )
tpath = ".. /input/test"
files = os.listdir(tpath)
 i = 0
test_id = []
test_df = pd.DataFrame(dtype=np.float64, columns=df.columns.values[:-1])
 for f in tqdm_notebook(files):
seg = pd.read_csv(f'{tpath}/{f}')
 converted = get_counts(seg.acoustic_data.values)
 test_df.loc[i] = converted
test_id.append(f.replace(".csv", ""))
i+=1<define_variables>
test.loc[test['release_date'].isnull() == True, 'release_date'] = '01/01/98'
num = len(df)
 interval = int(num/100)
 values = int(num/(5*100))
valid_idx = []
for i in range(0,len(df)-values,interval):
for j in range(0,values-1):
valid_idx.append(i+j )<define_variables>
def fix_date(x):

 year = x.split('/')[2]
if int(year)<= 19:
return x[:-2] + '20' + year
else:
return x[:-2] + '19' + year
valid_ttfs = np.array([df.iloc[i].expected for i in valid_idx] )<categorify>
train['release_date'] = train['release_date'].apply(lambda x: fix_date(x))
test['release_date'] = test['release_date'].apply(lambda x: fix_date(x))
train['release_date'] = pd.to_datetime(train['release_date'])
 test['release_date'] = pd.to_datetime(test['release_date'] )
data = TabularDataBunch.from_df(path, df, "expected", valid_idx=valid_idx, test_df=test_df, procs=[Normalize])
<find_best_params>
def process_date(df):
date_parts = ["year", "weekday", "month", 'weekofyear', 'day', 'quarter']
for part in date_parts:
part_col = 'release_date' + "_" + part
df[part_col] = getattr(df['release_date'].dt, part ).astype(int)
 return df
train = process_date(train)
 test = process_date(test )
%%time
best_learn = None
best_mae = 9999
for i in range(0, 99):
learn = tabular_learner(data=data, layers=[200,100], metrics=mae, ps=0.5, y_range=(-1,15))
learn.callbacks = [SaveModelCallback(learn, every='improvement', mode='min', name='best')]
learn.fit_one_cycle(20, 1e-2)
 gc.collect()
preds = learn.get_preds(DatasetType.Valid)[0].numpy().flatten()
new_mae = np.abs(valid_ttfs-preds ).mean()
if new_mae < best_mae or not best_learn:
best_learn = learn
best_mae = new_mae
print(f'Run {i} - Best MAE: {best_mae}' )<predict_on_test>
train['status'].value_counts()
preds = best_learn.get_preds(DatasetType.Test)[0].numpy().flatten()<define_variables>
test['status'].value_counts()
tpath = ".. /input/test"
files = os.listdir(tpath)
 files = [f.replace(".csv","")for f in files]
files[:3]<prepare_output>
train = train.drop(['homepage', 'imdb_id', 'poster_path', 'release_date', 'status', 'log_revenue'], axis=1)
 test = test.drop(['homepage', 'imdb_id', 'poster_path', 'release_date', 'status'], axis=1 )
results = pd.DataFrame({"seg_id":files, "time_to_failure":preds})
 results.head()<save_to_csv>
for col in train.columns:
if train[col].nunique() == 1:
print(col)
 train = train.drop([col], axis=1)
 test = test.drop([col], axis=1 )
results.to_csv('submission.csv',index=False )<set_options>
for col in ['original_language', 'collection_name', 'all_genres']:
le = LabelEncoder()
le.fit(list(train[col].fillna('')) + list(test[col].fillna('')))
 train[col] = le.transform(train[col].fillna('' ).astype(str))
test[col] = le.transform(test[col].fillna('' ).astype(str))
warnings.filterwarnings("ignore")
 BLUE = '
RED = '
GREEN = ' 
<define_variables>
train_texts = train[['title', 'tagline', 'overview', 'original_title']]
test_texts = test[['title', 'tagline', 'overview', 'original_title']]
Image(".. /input/feature-generation-frame/feature_generation_framework.png" )<load_from_csv>
for col in ['title', 'tagline', 'overview', 'original_title']:
train['len_' + col] = train[col].fillna('' ).apply(lambda x: len(str(x)))
 train['words_' + col] = train[col].fillna('' ).apply(lambda x: len(str(x.split(' '))))
train = train.drop(col, axis=1)
 test['len_' + col] = test[col].fillna('' ).apply(lambda x: len(str(x)))
 test['words_' + col] = test[col].fillna('' ).apply(lambda x: len(str(x.split(' '))))
test = test.drop(col, axis=1 )
train = pd.read_csv(".. /input/processed/train_data.csv")
 test = pd.read_csv(".. /input/processed/test_data.csv" )<feature_engineering>
train.loc[train['id'] == 16,'revenue'] = 192864
train.loc[train['id'] == 90,'budget'] = 30000000
train.loc[train['id'] == 118,'budget'] = 60000000
train.loc[train['id'] == 149,'budget'] = 18000000
train.loc[train['id'] == 313,'revenue'] = 12000000
train.loc[train['id'] == 451,'revenue'] = 12000000
train.loc[train['id'] == 464,'budget'] = 20000000
train.loc[train['id'] == 470,'budget'] = 13000000
train.loc[train['id'] == 513,'budget'] = 930000
train.loc[train['id'] == 797,'budget'] = 8000000
train.loc[train['id'] == 819,'budget'] = 90000000
train.loc[train['id'] == 850,'budget'] = 90000000
train.loc[train['id'] == 1112,'budget'] = 7500000
train.loc[train['id'] == 1131,'budget'] = 4300000
train.loc[train['id'] == 1359,'budget'] = 10000000
train.loc[train['id'] == 1542,'budget'] = 1
train.loc[train['id'] == 1570,'budget'] = 15800000
train.loc[train['id'] == 1571,'budget'] = 4000000
train.loc[train['id'] == 1714,'budget'] = 46000000
train.loc[train['id'] == 1721,'budget'] = 17500000
train.loc[train['id'] == 1865,'revenue'] = 25000000
train.loc[train['id'] == 2268,'budget'] = 17500000
train.loc[train['id'] == 2491,'revenue'] = 6800000
train.loc[train['id'] == 2602,'budget'] = 31000000
train.loc[train['id'] == 2612,'budget'] = 15000000
train.loc[train['id'] == 2696,'budget'] = 10000000
train.loc[train['id'] == 2801,'budget'] = 10000000
test.loc[test['id'] == 3889,'budget'] = 15000000
test.loc[test['id'] == 6733,'budget'] = 5000000
test.loc[test['id'] == 3197,'budget'] = 8000000
test.loc[test['id'] == 6683,'budget'] = 50000000
test.loc[test['id'] == 5704,'budget'] = 4300000
test.loc[test['id'] == 6109,'budget'] = 281756
test.loc[test['id'] == 7242,'budget'] = 10000000
test.loc[test['id'] == 7021,'budget'] = 17540562
test.loc[test['id'] == 5591,'budget'] = 4000000
test.loc[test['id'] == 4282,'budget'] = 20000000
power_six = train.id[train.budget > 1000][train.revenue < 100]
for k in power_six :
train.loc[train['id'] == k,'revenue'] = train.loc[train['id'] == k,'revenue'] * 1000000
train['fold'] = np.arange(len(train)) // 10
train.loc[(np.arange(len(train)) % 10 > 4), 'fold'] = -1
train = train[train.fold != -1].reset_index().drop('index', axis=1)
 N = train.fold.max()
K = 10
k = N // K
np.random.seed(0)
 permutation = np.random.choice(N, N, replace=False)
 folds = []
for i in range(K):
fold = permutation[i*k:(i+1)*k]
folds.append(fold)
 tmp = train[~np.isin(train.fold, fold)]<train_on_grid>
X = train.drop(['id', 'revenue'], axis=1)
 y = np.log1p(train['revenue'])
 X_test = test.drop(['id'], axis=1 )
def build_linear_model(feature):
train['prediction'] = 0
for i in range(K):
val_fold = folds[i]
cv_train = train[(~np.isin(train.fold, val_fold)) ]
cv_val = train[np.isin(train.fold, val_fold)]
feature_train = cv_train[feature].values
target_train = cv_train[TARGET].values
feature_val = cv_val[feature].values
target_val = cv_val[TARGET].values
model = LinearRegression()
model.fit(feature_train.reshape(-1,1), target_train)
 val_predictions = model.predict(feature_val.reshape(-1,1))
train.loc[np.isin(train.fold, val_fold), 'prediction'] = val_predictions
cv_train = train.copy()
feature_train = cv_train[feature].values
target_train = cv_train[TARGET].values
model = LinearRegression()
model.fit(feature_train.reshape(-1,1), target_train)
 score = mae(train[TARGET].values, train.prediction.values)
 fig, ax = plt.subplots(figsize=(18,6))
plt.subplot(121)
 plt.plot(train[TARGET].values, c=RED, label='true')
 plt.plot(train.prediction.values, c=BLUE, alpha=0.5, label='prediction')
 plt.xlabel('time')
 plt.ylabel('time_to_failure')
 plt.legend()
plt.grid()
plt.subplot(122)
 plt.scatter(y=train[TARGET].values, x=feature_train, c=BLUE, alpha=0.1, label='true')
 a = np.quantile(feature_train, 0.05)
 b = np.quantile(feature_train, 0.95)
 x = np.linspace(a, b, 10)
 features = np.vstack([x] ).T
preds = model.predict(features.reshape(-1,1))
plt.plot(features, preds, c=RED, alpha=1., label='linear model')
 plt.xlabel(feature)
 plt.ylabel('time_to_failure')
 plt.legend()
plt.grid()
print("MAE =", score)
 plt.show()
return score
TARGET = 'time_to_failure'
for feature in train.columns[:-3]:
build_linear_model(feature )<choose_model_class>
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.1 )
feature = 'roll_100_std_percentile_5'
feature_values = train[feature].values
threshold =.32
target = train[TARGET]
index1 =(target > threshold)
 index2 =(target <= threshold)
 model1 = LinearRegression()
model1.fit(feature_values[index1].reshape(-1,1), train[TARGET].values[index1])
 model2 = LinearRegression()
model2.fit(feature_values[index2].reshape(-1,1), train[TARGET].values[index2])
 preds1 = model1.predict(feature_values.reshape(-1,1))
preds1[preds1 < 0] = 0
preds2 = model2.predict(feature_values.reshape(-1,1))
preds2[preds2 < 0] = 0
preds = model1.predict(feature_values.reshape(-1,1))
preds[index2] = preds2[index2]
preds[preds < 0] = 0
score = mae(train[TARGET].values, preds)
 print("MAE =", score)
 fig, ax = plt.subplots(figsize=(18,6))
plt.subplot(131)
 plt.plot(target, c=RED, label='true')
 plt.plot(preds, c=BLUE, alpha=0.5, label='prediction')
 plt.title('Two linear models')
 plt.xlabel('time')
 plt.ylabel('time_to_failure')
 plt.legend()
plt.grid()
plt.subplot(132)
 plt.scatter(y=target[index1], x=feature_values[index1], c=BLUE, alpha=0.1, label='true')
 a = np.quantile(feature_values, 0.05)
 b = np.quantile(feature_values, 0.95)
 x = np.linspace(a, b, 10)
 features = np.vstack([x] ).T
preds1 = model1.predict(features.reshape(-1,1))
plt.plot(features, preds1, c=RED, alpha=1., label='linear model')
 plt.title('More than {} seconds to quake'.format(threshold))
plt.xlabel(feature)
 plt.ylabel('time_to_failure')
 plt.legend()
plt.grid()
plt.subplot(133)
 plt.scatter(y=target[index2], x=feature_values[index2], c=BLUE, alpha=0.1, label='true')
 a = np.quantile(feature_values, 0.05)
 b = np.quantile(feature_values, 0.95)
 x = np.linspace(a, b, 10)
 features = np.vstack([x] ).T
preds2 = model2.predict(features.reshape(-1,1))
plt.plot(features, preds2, c=RED, alpha=1., label='linear model')
 plt.title('Less than {} seconds to quake'.format(threshold))
plt.xlabel(feature)
 plt.ylabel('time_to_failure')
 plt.legend()
plt.grid()<define_variables>
params = {'num_leaves': 30,
'min_data_in_leaf': 20,
'objective': 'regression',
'max_depth': 5,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.2,
"verbosity": -1}
model1 = lgb.LGBMRegressor(**params, n_estimators = 20000, nthread = 4, n_jobs = -1)
 model1.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='rmse',
verbose=1000, early_stopping_rounds=200 )
train['classifier_32'] = 0
train['target'] = train.time_to_failure < 0.32
TARGET = 'target'
features = [
'mag_freq0_percentile_50',
'bp5_mad',
'bp1_interquantile_range_5_95',
'evol_bp2_interquantile_range_10_90',
'roll_4096_mean_std',
'roll_1000_std_percentile_5',
'bp0_kurtosis',
'bp1_percentile_10',
'evol_bp3_percentile_75'
]
params = {
'num_leaves': 32,
'max_bin': 63,
'min_data_in_leaf': 50,
'learning_rate': 0.05,
'min_sum_hessian_in_leaf': 0.01,
'bagging_fraction': 0.8,
'bagging_freq': 1,
'feature_fraction': 1,
'min_gain_to_split': 0.02,
'max_depth': 6,
'save_binary': True,
'seed': 0,
'feature_fraction_seed': 0,
'bagging_seed': 0,
'drop_seed': 0,
'data_random_seed': 0,
'objective': 'binary',
'boosting_type': 'gbdt',
'verbose': 1,
'boost_from_average': True,
'metric': 'auc',
'is_unbalance': True
}
num_boost_round=100
for i in range(K):
val_fold = folds[i]
cv_train = train[(~np.isin(train.fold, val_fold)) ]
cv_val = train[np.isin(train.fold, val_fold)]
xg_train = lgb.Dataset(cv_train[features].values, label=cv_train[TARGET].values)
 xg_val = lgb.Dataset(cv_val[features].values, label=cv_val[TARGET].values)
 model = lgb.train(params, xg_train, num_boost_round=num_boost_round, verbose_eval=0)
 val_predictions = model.predict(cv_val[features].values, num_iteration=num_boost_round)
 train.loc[np.isin(train.fold, val_fold), 'classifier_32'] = val_predictions
xg_train = lgb.Dataset(train[features].values, label=train[TARGET].values)
 model = lgb.train(params, xg_train, num_boost_round=num_boost_round)
 test['classifier_32'] = model.predict(test[features].values, num_iteration=num_boost_round)
 fpr, tpr, _ = roc_curve(train['target'], train['classifier_32'])
 roc_auc = auc(fpr, tpr)
 plt.figure(figsize=(8,8))
lw = 2
plt.plot(fpr, tpr, color=BLUE,
lw=lw, label='ROC curve(area = %0.2f)' % roc_auc)
 plt.plot([0, 1], [0, 1], color=RED, lw=lw, linestyle='--')
 plt.xlim([0.0, 1.0])
 plt.ylim([0.0, 1.05])
 plt.xlabel('False Positive Rate')
 plt.ylabel('True Positive Rate')
 plt.title('Classifier ROC curve')
 plt.legend(loc="lower right")
 plt.show()<init_hyperparams>
eli5.show_weights(model1, feature_filter=lambda x: x != '<BIAS>' )
features = [
'roll_2048_std_percentile_10',
'roll_4096_mean_percentile_95',
'bp6_percentile_90',
'bp2_interquantile_range_20_80',
'bp5_percentile_99',
'bp7_interquantile_range_10_90',
'bp4_percentile_80',
'roll_100_mean_mean',
'roll_1000_mean_percentile_99',
'roll_100_std_percentile_5',
'bp7_mad',
'evol_bp5_min',
'evol_bp7_max'
]
params = {'num_leaves': 5,
'max_bin': 63,
'min_data_in_leaf': 20,
'learning_rate': 0.05,
'min_sum_hessian_in_leaf': 0.008,
'bagging_fraction': 0.9,
'bagging_freq': 1,
'feature_fraction': 1.,
'min_gain_to_split': 0.5,
'max_depth': 4,
'save_binary': True,
'seed': 0,
'feature_fraction_seed': 0,
'bagging_seed': 0,
'drop_seed': 0,
'data_random_seed': 0,
'objective': 'huber',
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'mae',
'boost_from_average': True,
}
num_rounds = 2000
early_stopping = 50
TARGET = 'time_to_failure'
scores_train = []
scores_val = []
count = 0
test['lgbm_prediction'] = 0
for i in range(K):
val_fold = folds[i]
cv_train = train[(~np.isin(train.fold, val_fold)) &(train.time_to_failure > 0.32)]
cv_val = train[np.isin(train.fold, val_fold)&(train.time_to_failure > 0.0)]
xg_train = lgb.Dataset(cv_train[features].values, label=cv_train[TARGET].values)
 model = lgb.train(params, xg_train, num_rounds, verbose_eval=0)
 train_preds = model.predict(cv_train[features].values, num_iteration=num_rounds)
 val_preds = model.predict(cv_val[features].values, num_iteration=num_rounds)
 train.loc[np.isin(train.fold, val_fold), 'lgbm_prediction'] = val_preds
scores_train.append(mae(cv_train[TARGET].values, train_preds))
scores_val.append(mae(cv_val[TARGET].values, val_preds))
count += 1
test['lgbm_prediction'] += model.predict(test[features].values, num_iteration=num_rounds)
 test['lgbm_prediction'] = test['lgbm_prediction'] / count
print("MAE =", mae(train.time_to_failure[:-5], train.lgbm_prediction[:-5]))
fig, ax = plt.subplots(figsize=(18,6))
plt.subplot(111)
 plt.plot(train[TARGET].values, c=RED, label='true')
 plt.plot(train.lgbm_prediction.values, c=BLUE, alpha=0.5, label='prediction')
 plt.title('LightGBM predictions')
 plt.xlabel('time')
 plt.ylabel('time_to_failure')
 plt.legend()
plt.grid()
plt.show()<feature_engineering>
n_fold = 5
folds = KFold(n_splits=n_fold, shuffle=True, random_state=42 )
feature = 'roll_100_std_percentile_5'
scores_train = []
scores_val = []
count = 0
test['nusvr_prediction'] = 0
threshold = 0.32
for i in range(K):
val_fold = folds[i]
cv_train = train[(~np.isin(train.fold, val_fold)) &(train.time_to_failure <= threshold)]
cv_val = train[np.isin(train.fold, val_fold)&(train.time_to_failure > 0.0)]
feature_train = cv_train[feature].values
target_train = cv_train[TARGET].values
feature_val = cv_val[feature].values
target_val = cv_val[TARGET].values
feature_test = test[feature].values
params = {
'C': 30,
'gamma': 0.05,
'kernel': 'rbf',
'nu': 1,
}
model = NuSVR(**params)
 model.fit(feature_train.reshape(-1,1), 1/(threshold + 0.01 - target_train))
train_preds = threshold + 0.01 - 1/model.predict(feature_train.reshape(-1,1))
train_preds[train_preds < 0] = 0
val_preds = threshold + 0.01 - 1/model.predict(feature_val.reshape(-1,1))
val_preds[val_preds < 0] = 0
test_preds = threshold + 0.01 - 1/model.predict(feature_test.reshape(-1,1))
test_preds[test_preds < 0] = 0
train.loc[np.isin(train.fold, val_fold), 'nusvr_prediction'] = val_preds
scores_train.append(mae(cv_train[TARGET].values, train_preds))
scores_val.append(mae(cv_val[TARGET].values, val_preds))
count += 1
test['nusvr_prediction'] += test_preds
test['nusvr_prediction'] = test['nusvr_prediction'] / count
fig, ax = plt.subplots(figsize=(18,6))
plt.subplot(111)
 plt.plot(train[TARGET].values, c=RED, label='true')
 plt.plot(train.nusvr_prediction.values, c=BLUE, alpha=0.5, label='prediction')
 plt.title('NuSVR predictions')
 plt.xlabel('time')
 plt.ylabel('time_to_failure')
 plt.legend()
plt.grid()
plt.show()<categorify>
def train_model(X, X_test, y, params=None, folds=folds, model_type='lgb', plot_feature_importance=False, model=None):
oof = np.zeros(X.shape[0])
 prediction = np.zeros(X_test.shape[0])
 scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(folds.split(X)) :
print('Fold', fold_n, 'started at', time.ctime())
 if model_type == 'sklearn':
X_train, X_valid = X[train_index], X[valid_index]
else:
X_train, X_valid = X.values[train_index], X.values[valid_index]
y_train, y_valid = y[train_index], y[valid_index]
if model_type == 'lgb':
model = lgb.LGBMRegressor(**params, n_estimators = 20000, nthread = 4, n_jobs = -1)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='rmse',
verbose=1000, early_stopping_rounds=200)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration_)
 if model_type == 'xgb':
train_data = xgb.DMatrix(data=X_train, label=y_train)
 valid_data = xgb.DMatrix(data=X_valid, label=y_valid)
 watchlist = [(train_data, 'train'),(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=20000, evals=watchlist, early_stopping_rounds=200, verbose_eval=500, params=params)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid), ntree_limit=model.best_ntree_limit)
 y_pred = model.predict(xgb.DMatrix(X_test.values), ntree_limit=model.best_ntree_limit)
 if model_type == 'sklearn':
model = model
model.fit(X_train, y_train)
 y_pred_valid = model.predict(X_valid ).reshape(-1,)
 score = mean_squared_error(y_valid, y_pred_valid)
 y_pred = model.predict(X_test)
 if model_type == 'cat':
model = CatBoostRegressor(iterations=20000, eval_metric='RMSE', **params)
 model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=False)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test)
 oof[valid_index] = y_pred_valid.reshape(-1,)
 scores.append(mean_squared_error(y_valid, y_pred_valid)** 0.5)
 prediction += y_pred
if model_type == 'lgb':
fold_importance = pd.DataFrame()
fold_importance["feature"] = X.columns
fold_importance["importance"] = model.feature_importances_
fold_importance["fold"] = fold_n + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 prediction /= n_fold
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 if model_type == 'lgb':
feature_importance["importance"] /= n_fold
if plot_feature_importance:
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
return oof, prediction, feature_importance
return oof, prediction
else:
return oof, prediction
data = []
for x in np.linspace(0.,1.,21):
def mix(row):
t = x
if row.classifier_32 < t:
return row.lgbm_prediction
else:
return row.nusvr_prediction
train['final_prediction'] = train.apply(mix, axis=1)
 data.append([np.round(x,2), mae(train.time_to_failure[:-5], train.final_prediction[:-5])])
 pd.DataFrame(data=data, columns=['threshold', 'MAE'] )<save_to_csv>
params = {'num_leaves': 30,
'min_data_in_leaf': 10,
'objective': 'regression',
'max_depth': 5,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.2,
"verbosity": -1}
oof_lgb, prediction_lgb, _ = train_model(X, X_test, y, params=params, model_type='lgb', plot_feature_importance=True )
submission = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv')
 submission = submission.drop('time_to_failure', axis=1)
 preds = test[['time_to_failure', 'seg_id']]
submission = submission.merge(preds, how='left')
 submission.to_csv('submission_nusvr_lgbm_classifier.csv', index=False)
 submission.head()<concatenate>
for col in train_texts.columns:
vectorizer = TfidfVectorizer(
 sublinear_tf=True,
analyzer='word',
token_pattern=r'\w{1,}',
ngram_range=(1, 2),
min_df=10 
)
 vectorizer.fit(list(train_texts[col].fillna('')) + list(test_texts[col].fillna('')))
 train_col_text = vectorizer.transform(train_texts[col].fillna(''))
test_col_text = vectorizer.transform(test_texts[col].fillna(''))
model = linear_model.RidgeCV(alphas=(0.01, 0.1, 1.0, 10.0, 100.0), scoring='neg_mean_squared_error', cv=folds)
 oof_text, prediction_text = train_model(train_col_text, test_col_text, y, params=None, model_type='sklearn', model=model)
 X[col + '_oof'] = oof_text
X_test[col + '_oof'] = prediction_text
samp_per_second = 31
ttf_array = np.concatenate([np.linspace(11, 0, samp_per_second * 11),
np.linspace(11, 0, samp_per_second * 11),
np.linspace(11, 0, samp_per_second * 11),
np.linspace(5, 0, samp_per_second * 5),
np.linspace(10, 0, samp_per_second * 10),
np.linspace(15, 0, samp_per_second * 15),
np.linspace(7, 0, samp_per_second * 7),
np.linspace(15, 10, samp_per_second * 5)])
 ss = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv')
 ss_simulated = pd.DataFrame(ttf_array, columns=['time_to_failure'])
 ss_simulated.plot(figsize=(15, 5))<load_from_csv>
def new_features(df):
df['budget_to_popularity'] = df['budget'] / df['popularity']
df['budget_to_runtime'] = df['budget'] / df['runtime']
df['_budget_year_ratio'] = df['budget'] /(df['release_date_year'] * df['release_date_year'])
 df['_releaseYear_popularity_ratio'] = df['release_date_year'] / df['popularity']
df['_releaseYear_popularity_ratio2'] = df['popularity'] / df['release_date_year']
df['runtime_to_mean_year'] = df['runtime'] / df.groupby("release_date_year")["runtime"].transform('mean')
 df['popularity_to_mean_year'] = df['popularity'] / df.groupby("release_date_year")["popularity"].transform('mean')
 df['budget_to_mean_year'] = df['budget'] / df.groupby("release_date_year")["budget"].transform('mean')
 return df
sub1352 = pd.read_csv('.. /input/lanl-submissions/LB1.352_submission_147.csv')
 sub1389 = pd.read_csv('.. /input/lanl-submissions/LB1.389_submissionverson_one.csv')
 sub1362 = pd.read_csv('.. /input/lanl-submissions/LB1.362_submission_147.csv')
 sub1371 = pd.read_csv('.. /input/lanl-submissions/LB1.371_submission_137.csv')
 sub1380 = pd.read_csv('.. /input/lanl-submissions/LB1.380_submission.csv')
 sub1381 = pd.read_csv('.. /input/lanl-submissions/LB1.381_submission_147.csv')
 sub1389 = pd.read_csv('.. /input/lanl-submissions/LB1.389_submissionverson_one.csv')
 sub_50feats = pd.read_csv('.. /input/lanl-submissions/submission_50feats_peaks 9000ks.csv')
 ss['sub1.352'] = sub1352['time_to_failure']
ss['sub1.389'] = sub1389['time_to_failure']
ss['sub1.362'] = sub1362['time_to_failure']
ss['sub1.371'] = sub1371['time_to_failure']
ss['sub1.380'] = sub1380['time_to_failure']
ss['sub1.381'] = sub1381['time_to_failure']
ss['sub1.389'] = sub1389['time_to_failure']
ss['simulated'] = ss_simulated['time_to_failure']<groupby>
X = new_features(X)
 X_test = new_features(X_test )
sub1389.agg(['mean','median'] )<load_from_csv>
oof_lgb, prediction_lgb, _ = train_model(X, X_test, y, params=params, model_type='lgb', plot_feature_importance=True )
sub1350_2037CV = pd.read_csv('.. /input/last-10-subs-data/1350_submission_lgb_FINALSHUFFLED_2.037CV_123Feats_24000Rows.csv')
 sub1355_2038CV = pd.read_csv('.. /input/last-10-subs-data/1355_submission_lgb_FINALSHUFFLED_2.038CV_119Feats_24000Rows.csv')
 sub1383_1991CV = pd.read_csv('.. /input/last-10-subs-data/1383_submission_lgb_FINALSHUFFLED_1.991CV_120Feats_24000Rows.csv')
 sub1388_2086CV = pd.read_csv('.. /input/last-10-subs-data/1398_submission_lgb_FINALSHUFFLED_2.086CV_138Feats_24000Rows.csv')
 sub1404_2009CV = pd.read_csv('.. /input/last-10-subs-data/1404_submission_lgb_FINALSHUFFLED_2.009CV_116Feats_24000Rows.csv')
 sub1432_xgb = pd.read_csv('.. /input/last-10-subs-data/1432_submission_xgb_FINALSHUFFLED.csv')
 sub1529_xgb = pd.read_csv('.. /input/last-10-subs-data/1529_submission_xgb_FINALSHUFFLED.csv')
 sub1532_1948CV = pd.read_csv('.. /input/last-10-subs-data/1532_submission_lgb_1.948CV_110Feats_24000Rows_15Fold.csv')
 sub1534_2092CV = pd.read_csv('.. /input/last-10-subs-data/1534_submission_lgb_FINALSHUFFLED_2.092CV_24Feats_24000Rows.csv')
 sub1863_2283CV = pd.read_csv('.. /input/last-10-subs-data/1863_submission_lgb_FINALSHUFFLED_2.283CV_17Feats_24000Rows.csv')
 ss['sub1350_2037CV'] = sub1350_2037CV['time_to_failure']
ss['sub1355_2038CV'] = sub1355_2038CV['time_to_failure']
ss['sub1383_1991CV'] = sub1383_1991CV['time_to_failure']
ss['sub1388_2086CV'] = sub1388_2086CV['time_to_failure']
ss['sub1404_2009CV'] = sub1404_2009CV['time_to_failure']
ss['sub1432_xgb'] = sub1432_xgb['time_to_failure']
ss['sub1529_xgb'] = sub1529_xgb['time_to_failure']
ss['sub1532_1948CV'] = sub1532_1948CV['time_to_failure']
ss['sub1534_2092CV'] = sub1534_2092CV['time_to_failure']
ss['sub1863_2283CV'] = sub1863_2283CV['time_to_failure']<load_from_csv>
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.1)
 params = {'num_leaves': 30,
'min_data_in_leaf': 20,
'objective': 'regression',
'max_depth': 7,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.2,
"verbosity": -1}
model1 = lgb.LGBMRegressor(**params, n_estimators = 20000, nthread = 4, n_jobs = -1)
 model1.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='rmse',
verbose=1000, early_stopping_rounds=200)
 eli5.show_weights(model1, feature_filter=lambda x: x != '<BIAS>' )
submit = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv' )<feature_engineering>
def top_cols_interaction(df):
df['budget_to_year'] = df['budget'] / df['release_date_year']
df['budget_to_mean_year_to_year'] = df['budget_to_mean_year'] / df['release_date_year']
df['popularity_to_mean_year_to_log_budget'] = df['popularity_to_mean_year'] / df['log_budget']
df['year_to_log_budget'] = df['release_date_year'] / df['log_budget']
df['budget_to_runtime_to_year'] = df['budget_to_runtime'] / df['release_date_year']
df['genders_1_cast_to_log_budget'] = df['genders_1_cast'] / df['log_budget']
df['all_genres_to_popularity_to_mean_year'] = df['all_genres'] / df['popularity_to_mean_year']
df['genders_2_crew_to_budget_to_mean_year'] = df['genders_2_crew'] / df['budget_to_mean_year']
df['overview_oof_to_genders_2_crew'] = df['overview_oof'] / df['genders_2_crew']
return df
submit['time_to_failure'] = ss[['sub1.380','sub1.381','sub1529_xgb','sub1532_1948CV','sub1432_xgb']].mean(axis=1 )<save_to_csv>
X = top_cols_interaction(X)
 X_test = top_cols_interaction(X_test )
submit.to_csv('blend_5_including2xgb.csv', index=False )<set_options>
X = X.replace([np.inf, -np.inf], 0 ).fillna(0)
 X_test = X_test.replace([np.inf, -np.inf], 0 ).fillna(0 )
warnings.filterwarnings("ignore")
 warnings.simplefilter(action='ignore', category=FutureWarning)
<load_from_csv>
trainAdditionalFeatures = pd.read_csv('.. /input/tmdb-competition-additional-features/TrainAdditionalFeatures.csv')
 testAdditionalFeatures = pd.read_csv('.. /input/tmdb-competition-additional-features/TestAdditionalFeatures.csv')
 train = pd.read_csv('.. /input/tmdb-box-office-prediction/train.csv')
 test = pd.read_csv('.. /input/tmdb-box-office-prediction/test.csv')
 X['imdb_id'] = train['imdb_id']
X_test['imdb_id'] = test['imdb_id']
del train, test
X = pd.merge(X, trainAdditionalFeatures, how='left', on=['imdb_id'])
 X_test = pd.merge(X_test, testAdditionalFeatures, how='left', on=['imdb_id'])
 X = X.drop(['imdb_id'], axis=1)
 X_test = X_test.drop(['imdb_id'], axis=1 )
train_X_0 = pd.read_csv(".. /input/lanl-master-s-features-creating-0/train_X_features_865.csv")
 train_X_1 = pd.read_csv(".. /input/lanl-master-s-features-creating-1/train_X_features_865.csv")
 y_0 = pd.read_csv(".. /input/lanl-master-s-features-creating-0/train_y.csv", index_col=False, header=None)
 y_1 = pd.read_csv(".. /input/lanl-master-s-features-creating-1/train_y.csv", index_col=False, header=None )<concatenate>
params = {'num_leaves': 30,
'min_data_in_leaf': 20,
'objective': 'regression',
'max_depth': 9,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.2,
"verbosity": -1}
oof_lgb, prediction_lgb, _ = train_model(X, X_test, y, params=params, model_type='lgb', plot_feature_importance=True )
train_X = pd.concat([train_X_0, train_X_1], axis=0)
 train_X = train_X.reset_index(drop=True)
 print(train_X.shape)
 train_X.head()<concatenate>
xgb_params = {'eta': 0.01,
'objective': 'reg:linear',
'max_depth': 5,
'subsample': 0.8,
'colsample_bytree': 0.7,
'eval_metric': 'rmse',
'seed': 11,
'silent': True}
oof_xgb, prediction_xgb = train_model(X, X_test, y, params=xgb_params, model_type='xgb', plot_feature_importance=False )
train_y = pd.concat([y_0, y_1], axis=0)
 train_y = train_y.reset_index(drop=True)
 train_y.columns = ['time_to_failure']
train_y['time_to_failure'].shape<load_from_csv>
cat_params = {'learning_rate': 0.02,
'depth': 5,
'l2_leaf_reg': 10,
'bootstrap_type': 'Bernoulli',
'od_type': 'Iter',
'od_wait': 50,
'random_seed': 11,
'allow_writing_files': False}
oof_cat, prediction_cat = train_model(X, X_test, y, params=cat_params, model_type='cat' )
test_X = pd.read_csv(".. /input/lanl-master-s-features-creating-0/test_X_features_10.csv" )<normalization>
params = {'num_leaves': 30,
'min_data_in_leaf': 20,
'objective': 'regression',
'max_depth': 3,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.2,
"verbosity": -1}
oof_lgb_1, prediction_lgb_1 = train_model(X, X_test, y, params=params, model_type='lgb', plot_feature_importance=False )
scaler = StandardScaler()
train_columns = train_X.columns
train_X[train_columns] = scaler.fit_transform(train_X[train_columns])
 test_X[train_columns] = scaler.transform(test_X[train_columns] )<set_options>
params = {'num_leaves': 30,
'min_data_in_leaf': 20,
'objective': 'regression',
'max_depth': 7,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.7,
"bagging_freq": 5,
"bagging_fraction": 0.7,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.2,
"verbosity": -1}
oof_lgb_2, prediction_lgb_2 = train_model(X, X_test, y, params=params, model_type='lgb', plot_feature_importance=False )
h2o.init(max_mem_size='16G' )<create_dataframe>
train_stack = np.vstack([oof_lgb, oof_xgb, oof_cat, oof_lgb_1, oof_lgb_2] ).transpose()
train_stack = pd.DataFrame(train_stack, columns=['lgb', 'xgb', 'cat', 'lgb_1', 'lgb_2'])
 test_stack = np.vstack([prediction_lgb, prediction_xgb, prediction_cat, prediction_lgb_1, prediction_lgb_2] ).transpose()
test_stack = pd.DataFrame(test_stack, columns=['lgb', 'xgb', 'cat', 'lgb_1', 'lgb_2'] )
train = h2o.H2OFrame(train_X)
 test = h2o.H2OFrame(test_X)
 y_train = h2o.H2OFrame(train_y )<feature_engineering>
model = linear_model.RidgeCV(alphas=(0.01, 0.1, 1.0, 10.0, 100.0), scoring='neg_mean_squared_error', cv=folds)
 oof_rcv_stack, prediction_rcv_stack = train_model(train_stack.values, test_stack.values, y, params=None, model_type='sklearn', model=model )
<train_model><EOS> 
sub = pd.read_csv('.. /input/tmdb-box-office-prediction/sample_submission.csv')
 sub['revenue'] = np.expm1(prediction_lgb)
 sub.to_csv("lgb.csv", index=False)
 sub['revenue'] = np.expm1(( prediction_lgb + prediction_xgb)/ 2)
 sub.to_csv("blend.csv", index=False)
 sub['revenue'] = np.expm1(( prediction_lgb + prediction_xgb + prediction_cat)/ 3)
 sub.to_csv("blend1.csv", index=False)
 sub['revenue'] = np.expm1(( prediction_lgb + prediction_xgb + prediction_cat + prediction_lgb_1)/ 4)
 sub.to_csv("blend2.csv", index=False)
 sub['revenue'] = np.expm1(( prediction_lgb + prediction_xgb + prediction_cat + prediction_lgb_1 + prediction_lgb_2)/ 5)
 sub.to_csv("blend3.csv", index=False)
 sub['revenue'] = prediction_lgb_stack
sub.to_csv("stack_lgb.csv", index=False)
 sub['revenue'] = prediction_rcv_stack
sub.to_csv("stack_rcv.csv", index=False)