Kaggle competition: TMDB Box Office Prediction
<SOS> metric: RMSLE Kaggle data source: tmdb-box-office-prediction<compute_train_metric>
pd.options.display.max_columns = None
warnings.simplefilter(action='ignore')
 stopwords_eng = set(stopwords.words('english'))
%matplotlib inline
@jit
def eval_gini(y_true, y_prob):
y_true = np.asarray(y_true)
 y_true = y_true[np.argsort(y_prob)]
ntrue = 0
gini = 0
delta = 0
n = len(y_true)
 for i in range(n-1, -1, -1):
y_i = y_true[i]
ntrue += y_i
gini += y_i * delta
delta += 1 - y_i
gini = 1 - 2 * gini /(ntrue *(n - ntrue))
return gini<categorify>
train = pd.read_hdf('.. /input/tmdb-preprocessed/train_kamal.h5')
 test = pd.read_hdf('.. /input/tmdb-preprocessed/test_kamal.h5' )
def gini_xgb(preds, dtrain):
labels = dtrain.get_label()
gini_score = -eval_gini(labels, preds)
 return [('gini', gini_score)]
def add_noise(series, noise_level):
return series *(1 + noise_level * np.random.randn(len(series)))
 def target_encode(trn_series=None,
val_series=None,
tst_series=None,
target=None,
min_samples_leaf=1,
smoothing=1,
noise_level=0):

 assert len(trn_series)== len(target)
 assert trn_series.name == tst_series.name
temp = pd.concat([trn_series, target], axis=1)
 averages = temp.groupby(by=trn_series.name)[target.name].agg(["mean", "count"])
 smoothing = 1 /(1 + np.exp(-(averages["count"] - min_samples_leaf)/ smoothing))
prior = target.mean()
averages[target.name] = prior *(1 - smoothing)+ averages["mean"] * smoothing
averages.drop(["mean", "count"], axis=1, inplace=True)
 ft_trn_series = pd.merge(
 trn_series.to_frame(trn_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=trn_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_trn_series.index = trn_series.index
ft_val_series = pd.merge(
 val_series.to_frame(val_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=val_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_val_series.index = val_series.index
ft_tst_series = pd.merge(
 tst_series.to_frame(tst_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=tst_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_tst_series.index = tst_series.index
return add_noise(ft_trn_series, noise_level), add_noise(ft_val_series, noise_level), add_noise(ft_tst_series, noise_level)
<load_from_csv>
def release_date_mapper(str_date):
if str(str_date)== 'nan':
return str_date
month, day, year_candidate = str_date.split('/')
 year = int(year_candidate)
 if year < 100:
if year <= 19:
year += 2000
else:
year += 1900
return month + '-' + day + '-' + str(year)
 test_orig = pd.read_csv('.. /input/tmdb-box-office-prediction/test.csv')
 test_orig['release_date'] = test_orig['release_date'].map(release_date_mapper)
 train_orig = pd.read_csv('.. /input/tmdb-box-office-prediction/train.csv')
 train_orig['release_date'] = train_orig['release_date'].map(release_date_mapper)
 train_orig['dataset'] = 'Train'
test_orig['dataset'] = 'Test'
data_orig = pd.concat(( train_orig, test_orig))
latest_release_date_test = pd.to_datetime(train_orig['release_date'] ).max()
print(f"Only {100*sum(pd.to_datetime(train_orig['release_date'])> latest_release_date_test)/len(train_orig):.1f} % of original training movies were release after the latest movie in test set." )
train_df = pd.read_csv('.. /input/train.csv', na_values="-1")
 test_df = pd.read_csv('.. /input/test.csv', na_values="-1" )<define_variables>
%%HTML
<a id=missing-zeros></a>
<img src=https://i.imgflip.com/2wazn1.jpg width="250" align="left">
train_features = [
"ps_car_13",
	"ps_reg_03",
	"ps_ind_05_cat",
	"ps_ind_03",
	"ps_ind_15",
	"ps_reg_02",
	"ps_car_14",
	"ps_car_12",
	"ps_car_01_cat",
	"ps_car_07_cat",
	"ps_ind_17_bin",
	"ps_car_03_cat",
	"ps_reg_01",
	"ps_car_15",
	"ps_ind_01",
	"ps_ind_16_bin",
	"ps_ind_07_bin",
	"ps_car_06_cat",
	"ps_car_04_cat",
	"ps_ind_06_bin",
	"ps_car_09_cat",
	"ps_car_02_cat",
	"ps_ind_02_cat",
	"ps_car_11",
	"ps_car_05_cat",
	"ps_calc_09",
	"ps_calc_05",
	"ps_ind_08_bin",
	"ps_car_08_cat",
	"ps_ind_09_bin",
	"ps_ind_04_cat",
	"ps_ind_18_bin",
	"ps_ind_12_bin",
	"ps_ind_14",
]
combs = [ 
('ps_reg_01', 'ps_car_02_cat'), 
('ps_reg_01', 'ps_car_04_cat'),
]<prepare_x_and_y>
%%HTML
<img src=https://memegenerator.net/img/instances/84246705/let-make-missing-values-nan-again.jpg width="300">
id_test = test_df['id'].values
id_train = train_df['id'].values
y = train_df['target']
start = time.time()
for n_c,(f1, f2)in enumerate(combs):
name1 = f1 + "_plus_" + f2
print('current feature %60s %4d in %5.1f'
%(name1, n_c + 1,(time.time() - start)/ 60), end='')
 print('\r' * 75, end='')
 train_df[name1] = train_df[f1].apply(lambda x: str(x)) + "_" + train_df[f2].apply(lambda x: str(x))
test_df[name1] = test_df[f1].apply(lambda x: str(x)) + "_" + test_df[f2].apply(lambda x: str(x))
lbl = LabelEncoder()
lbl.fit(list(train_df[name1].values)+ list(test_df[name1].values))
train_df[name1] = lbl.transform(list(train_df[name1].values))
test_df[name1] = lbl.transform(list(test_df[name1].values))
train_features.append(name1)
 X = train_df[train_features]
test_df = test_df[train_features]
f_cats = [f for f in X.columns if "_cat" in f] 
<define_variables>
data.loc[data['budget'] == 0, 'budget'] = np.nan
y_valid_pred = 0*y
y_test_pred = 0<choose_model_class>
data[(data['budget'] < 37)&(data['budget'] > 32)]
K = 5
kf = KFold(n_splits = K, random_state = 1, shuffle = True)
 np.random.seed(0 )<choose_model_class>
%%HTML
<img src=https://image.tmdb.org/t/p/original/RdsII0tG6vdF6GTaiHePFaAKVM.jpg width="250">
model = XGBClassifier(
 n_estimators=MAX_ROUNDS,
max_depth=4,
objective="binary:logistic",
learning_rate=LEARNING_RATE,
subsample=.8,
min_child_weight=6,
colsample_bytree=.8,
scale_pos_weight=1.6,
gamma=10,
reg_alpha=8,
reg_lambda=1.3,
)<split>
data_enriched_path = '.. /input/tmdb-preprocessed/data_tmdb_enriched_with_popularities.h5'
if not os.path.exists(data_enriched_path):
tmdb.API_KEY = 'YOUR_KEY_HERE'
search = tmdb.Search()
hits_cout = 0
missing_credits_mask = data['budget'].isnull()
total_number_of_movies_to_fill = sum(missing_credits_mask)
 for index, movie_row in tqdm(data[missing_credits_mask].iterrows() , total=total_number_of_movies_to_fill):
response = search.movie(query=movie_row['title'])if np.isnan(movie_row['release_year'])else search.movie(query=movie_row['title'], year=movie_row['release_year'])
 if response['total_results'] == 1:
movie = tmdb.Movies(response['results'][0]['id'])
 movie_info = movie.info()
if movie_info['budget'] != 0 and np.isnan(data.loc[index, 'budget']):
hits_cout += 1
data.loc[index, 'budget'] = movie_info['budget']
data['budget_adjusted'] =(data['budget'].map(lambda x: [x])+ data['release_year'].map(lambda x: [x])).map(lambda budget_year:
cpi.inflate(budget_year[0], budget_year[1], to=2018))
data['_budget_runtime_ratio'] = data['budget_adjusted']/data['runtime']
data['_budget_popularity_ratio'] = data['budget_adjusted']/data['popularity']
data['_budget_year_ratio'] = data['budget_adjusted']/(data['release_year']*data['release_year'])
 data['_budget_rating_ratio'] = data['budget_adjusted']/data['rating']
data['_budget_totalVotes_ratio'] = data['budget_adjusted']/data['totalVotes']
for i,(train_index, test_index)in enumerate(kf.split(train_df)) :
y_train, y_valid = y.iloc[train_index].copy() , y.iloc[test_index]
X_train, X_valid = X.iloc[train_index,:].copy() , X.iloc[test_index,:].copy()
X_test = test_df.copy()
print("
Fold ", i)
 for f in f_cats:
X_train[f + "_avg"], X_valid[f + "_avg"], X_test[f + "_avg"] = target_encode(
 trn_series=X_train[f],
val_series=X_valid[f],
tst_series=X_test[f],
target=y_train,
min_samples_leaf=200,
smoothing=10,
noise_level=0 
)
 if OPTIMIZE_ROUNDS:
eval_set=[(X_valid,y_valid)]
fit_model = model.fit(X_train, y_train,
eval_set=eval_set,
eval_metric=gini_xgb,
early_stopping_rounds=EARLY_STOPPING_ROUNDS,
verbose=False 
)
 print(" Best N trees = ", model.best_ntree_limit)
 print(" Best gini = ", model.best_score)
 else:
fit_model = model.fit(X_train, y_train)
 pred = fit_model.predict_proba(X_valid)[:,1]
print(" Gini = ", eval_gini(y_valid, pred))
y_valid_pred.iloc[test_index] = pred
y_test_pred += fit_model.predict_proba(X_test)[:,1]
del X_test, X_train, X_valid, y_train
y_test_pred /= K
print("
Gini for full training set:")
 eval_gini(y, y_valid_pred )<save_to_csv>
train_data = data[data['dataset'] == 'Train']
yearly_movie_counts = train_data['release_year'].value_counts()
single_movie_years = set(yearly_movie_counts.index[yearly_movie_counts == 1])
 train_data.loc[train_data['release_year'].isin(single_movie_years), 'release_year'] = 1
trn_indices_set, val_indices_set = [set(indices)for indices in train_test_split(train_data.index, stratify=train_data['release_year'])]
train_data.loc[train_data.index.isin(trn_indices_set), 'dataset'] = 'Train_train'
train_data.loc[train_data.index.isin(val_indices_set), 'dataset'] = 'Train_val'
compare_barplots_categorical(train_data[['budget_cat']], train_data['dataset'] == 'Train_train', train_data['dataset'] == 'Train_val',
n_categories_in_figure=21, binned_categories=True)
 plt.title('Budget train/val distributions, stratified by release year', fontsize=20 )
val = pd.DataFrame()
val['id'] = id_train
val['target'] = y_valid_pred.values
val.to_csv('xgb_valid.csv', float_format='%.6f', index=False )<save_to_csv>
def nans_proportion(pd_series, logical_index=None):
if logical_index is None:
logical_index = np.ones_like(pd_series, dtype=np.bool)
 return pd_series[logical_index].isnull().sum() /logical_index.sum()
print(f"In train proportion of missing values is: {100*nans_proportion(data['rating'], data['dataset']=='Train'):.2f} %.")
 print(f"In test proportion of missing values is: {100*nans_proportion(data['rating'], data['dataset']=='Test'):.2f} %." )
sub = pd.DataFrame()
sub['id'] = id_test
sub['target'] = y_test_pred
sub.to_csv('xgb_submit.csv', float_format='%.6f', index=False )<define_variables>
data_orig = data_orig.merge(data[['id', 'rating']], on='id', how='left')
 low_rating_proportion_trn_orig = 100 * data_orig[(data_orig['release_year'] > 1980)& 
(data_orig['dataset'] == 'Train')
 ].groupby('release_year')['rating'].agg(lambda x: sum(x < 3)/len(x))
low_rating_proportion_tst_orig = 100 * data_orig[(data_orig['release_year'] > 1980)& 
(data_orig['dataset'] == 'Test')
 ].groupby('release_year')['rating'].agg(lambda x: sum(x < 3)/len(x))
plt.figure(figsize=(7, 4))
plot_with_dots(plt, low_rating_proportion_trn_orig)
 plot_with_dots(plt, low_rating_proportion_tst_orig)
 plt.xlabel('Year of Movie Release', fontsize=15)
 plt.ylabel('Proportion of yearly produced movies [%]', fontsize=15)
 plt.legend(['Train orig', 'Test orig'])
 plt.ylim(0, low_rating_proportion_trn.max())
 plt.title('Low-rating movies proportion', fontsize=20 )
MAX_ROUNDS = 1200
OPTIMIZE_ROUNDS = False
LEARNING_RATE = 0.024<import_modules>
data.loc[(data['dataset'] == 'Train')&(data['release_year'] == 1990)&(data['rating'] < 3), ['title', 'imdb_id']]
import numpy as np
import pandas as pd
from catboost import CatBoostClassifier
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
from numba import jit
from sklearn import *
import lightgbm as lgb
from multiprocessing import *<compute_train_metric>
print(f"Number of duplicated movies: {len(data)- data['imdb_id'].isnull().sum() - data['imdb_id'].nunique() }.")
 data = data.iloc[data.astype(str ).drop_duplicates().index]
print(f"After the cleaning, number of duplicated movies: {len(data)- data['imdb_id'].isnull().sum() - data['imdb_id'].nunique() }." )
@jit
def eval_gini(y_true, y_prob):
y_true = np.asarray(y_true)
 y_true = y_true[np.argsort(y_prob)]
ntrue = 0
gini = 0
delta = 0
n = len(y_true)
 for i in range(n-1, -1, -1):
y_i = y_true[i]
ntrue += y_i
gini += y_i * delta
delta += 1 - y_i
gini = 1 - 2 * gini /(ntrue *(n - ntrue))
return gini<data_type_conversions>
data.loc[(data['dataset'] == 'Train')&(data['release_year'] == 1999)&(data['rating'] < 3), 'title'].iloc[:5]
def transform_df(df):
df = pd.DataFrame(df)
 dcol = [c for c in df.columns if c not in ['id','target']]
df['ps_car_13_x_ps_reg_03'] = df['ps_car_13'] * df['ps_reg_03']
df['negative_one_vals'] = np.sum(( df[dcol]==-1 ).values, axis=1)
 for c in dcol:
if '_bin' not in c:
df[c+str('_median_range')] =(df[c].values > d_median[c] ).astype(np.int)
 df[c+str('_mean_range')] =(df[c].values > d_mean[c] ).astype(np.int)
 for c in one_hot:
if len(one_hot[c])>2 and len(one_hot[c])< 7:
for val in one_hot[c]:
df[c+'_oh_' + str(val)] =(df[c].values == val ).astype(np.int)
 return df
def multi_transform(df):
p = Pool(cpu_count())
 df = p.map(transform_df, np.array_split(df, cpu_count()))
df = pd.concat(df, axis=0, ignore_index=True ).reset_index(drop=True)
 p.close() ; p.join()
return df
def gini_lgb(preds, dtrain):
y = list(dtrain.get_label())
 score = eval_gini(y, preds)/ eval_gini(y, y)
 return 'gini', score, True 
<load_from_csv>
train_data = data[data['dataset'] == 'Train']
yearly_movie_counts = train_data['release_year'].value_counts()
single_movie_years = set(yearly_movie_counts.index[yearly_movie_counts == 1])
 train_data.loc[train_data['release_year'].isin(single_movie_years), 'release_year'] = 1
trn_indices_set, val_indices_set = [set(indices)for indices in train_test_split(train_data.index, stratify=train_data['release_year'])]
train_data.loc[train_data.index.isin(trn_indices_set), 'dataset'] = 'Train_train'
train_data.loc[train_data.index.isin(val_indices_set), 'dataset'] = 'Train_val'
plt.figure(figsize=(8, 7))
bins = train_data['rating'].quantile(np.arange(0, 1.001, 0.05))
sns.distplot(train_data.loc[train_data['dataset'] == 'Train_train', 'rating'], bins=bins)
 sns.distplot(train_data.loc[train_data['dataset'] == 'Train_val', 'rating'], bins=bins)
 plt.title('Rating, train/val distributions', fontsize=18)
 plt.legend(['Train: train', 'Train: val'] )
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv' )<prepare_x_and_y>
importances = pd.DataFrame()
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=21)
 data_adversarial_validation = data.drop(['Keywords', 'belongs_to_collection', 'homepage', 'budget_cat', 'cast', 'crew',
'genres', 'imdb_id', 'id', 'original_title', 'overview', 'poster_path',
'production_companies', 'production_countries', 'spoken_languages',
'tagline', 'title', 'original_language', 'release_date', 'status', 'revenue'] +
[col for col in data.columns if 'rating' in col.lower() or 'budget' in col.lower() ], axis=1)
 dataset_year = data_adversarial_validation['dataset'].map(str)+ '_' + data_adversarial_validation['release_year'].map(str)
 data_adversarial_validation['dataset'] = data_adversarial_validation['dataset'].map(lambda x: 1 if x == 'Test' else 0)
 auc_mean = 0
for fold_idx,(trn_idx, val_idx)in enumerate(skf.split(dataset_year, dataset_year)) :
df_trn = data_adversarial_validation.iloc[trn_idx]
trn_y = df_trn['dataset']
df_trn.drop(['dataset'], axis=1, inplace=True)
 df_val = data_adversarial_validation.iloc[val_idx]
val_y = df_val['dataset']
df_val.drop(['dataset'], axis=1, inplace=True)
 clf = LGBMClassifier(objective='binary',
boosting_type='gbdt',
n_jobs=-1,
n_estimators=2000,
metric='auc')
 clf.fit(
 df_trn, trn_y,
eval_set=[(df_trn, trn_y),(df_val, val_y)],
early_stopping_rounds=50,
verbose=0 
)
 auc_mean += roc_auc_score(val_y, clf.predict_proba(df_val)[:, 1])/skf.get_n_splits()
imp_df = pd.DataFrame({
'feature': df_trn.columns,
'gain': clf.feature_importances_,
'fold': [fold_idx + 1] * len(df_trn.columns),
})
 importances = pd.concat([importances, imp_df], axis=0, sort=False)
 print(f'Mean AUC: {auc_mean:.3}.' )
col = [c for c in train_df.columns if c not in ['id','target']]
col = [c for c in col if not c.startswith('ps_calc_')]
id_test = test_df['id'].values
id_train = train_df['id'].values
y = train_df['target']
X = train_df[col]
y_valid_pred = 0*y
X_test = test_df.drop(['id'], axis=1)
 y_test_pred = 0<choose_model_class>
print(f"In train proportion of missing values is: {100*nans_proportion(data['popularity2'], data['dataset']=='Train'):.2f} %.")
 print(f"In test proportion of missing values is: {100*nans_proportion(data['popularity2'], data['dataset']=='Test'):.2f} %." )
K = 5
kf = KFold(n_splits = K, random_state = 1, shuffle = True )<init_hyperparams>
print(f )
params = {
'learning_rate': LEARNING_RATE,
'max_depth': 4,
'lambda_l1': 16.7,
'boosting': 'gbdt',
'objective': 'binary',
'metric': 'auc',
'feature_fraction':.7,
'is_training_metric': False,
'seed': 99
}<categorify>
print(f"From the most recent added movies {100 * nans_proportion(data['overview'], data['release_date'] > latest_release_date_test):.1f} % do not have overview.")
 print(f"In the original dataset {100 * nans_proportion(data_orig['overview']):.1f} % of movies do not have overview" )
for i,(train_index, test_index)in enumerate(kf.split(train_df)) :
y_train, y_valid = y.iloc[train_index].copy() , y.iloc[test_index].copy()
X_train, X_valid = X.iloc[train_index,:].copy() , X.iloc[test_index,:].copy()
test = test_df.copy() [col]
print("
Fold ", i)
 one_hot = {c: list(X_train[c].unique())for c in X_train.columns}
X_train = X_train.replace(-1, np.NaN)
 d_median = X_train.median(axis=0)
 d_mean = X_train.mean(axis=0)
 X_train = X_train.fillna(-1)
 X_train = multi_transform(X_train)
 X_valid = multi_transform(X_valid)
 test = multi_transform(test)
 if OPTIMIZE_ROUNDS:
fit_model = lgb.train(
 params,
lgb.Dataset(X_train, label=y_train),
MAX_ROUNDS,
lgb.Dataset(X_valid, label=y_valid),
verbose_eval=50,
feval=gini_lgb,
early_stopping_rounds=200 
)
 print(" Best iteration = ", fit_model.best_iteration)
 pred = fit_model.predict(X_valid, num_iteration=fit_model.best_iteration)
 test_pred = fit_model.predict(test[col], num_iteration=fit_model.best_iteration)
 else:
fit_model = lgb.train(
 params,
lgb.Dataset(X_train, label=y_train),
MAX_ROUNDS,
verbose_eval=50 
)
 pred = fit_model.predict(X_valid)
 test_pred = fit_model.predict(test)
 print(" Gini = ", eval_gini(y_valid, pred))
y_valid_pred.iloc[test_index] =(np.exp(pred)- 1.0 ).clip(0,1)
 y_test_pred +=(np.exp(test_pred)- 1.0 ).clip(0,1)
 y_test_pred /= K
print("
Gini for full training set:")
 eval_gini(y, y_valid_pred )<save_to_csv>
pd.set_option('display.max_colwidth', -1)
 data.loc[(data['release_year'] > 2015)&(data['overview_word_count'] < 20), ['title', 'overview']].sample(5, random_state=2 )
val = pd.DataFrame()
val['id'] = id_train
val['target'] = y_valid_pred.values
val.to_csv('lgb_valid.csv', float_format='%.6f', index=False )<save_to_csv>
print(f"From originally available movies {100 * nans_proportion(data_orig['tagline']):.1f} % do not have a tagline.")
 print(f"From the most recent added movies {100 * nans_proportion(data['tagline'], data['release_date'] > latest_release_date_test):.1f} % do not have a tagline." )
sub = pd.DataFrame()
sub['id'] = id_test
sub['target'] = y_test_pred
sub.to_csv('lgb_submit.csv', float_format='%.6f', index=False )<set_options>
%%HTML
<img src=http://wist.info/wp/wp-content/uploads/2016/04/Chekhov-brevity-sister-of-talent-wist_info-quote.jpg width='400'>
warnings.filterwarnings("ignore")
 warnings.simplefilter(action='ignore', category=FutureWarning)
<load_from_csv>
print(f"From originally available movies {100 * nans_proportion(data_orig['crew']):.1f} % do not have information about crew.")
 fraction_of_record_with_missing_crew = data.loc[data['release_date'] > latest_release_date_test, 'crew'].map(lambda x: len(x)== 0 ).sum() /sum(data['release_date'] > latest_release_date_test)
 print(f'From the most recent added movies {100 * fraction_of_record_with_missing_crew:.1f} % do not have information about crew.' )
X_train_1 = pd.read_csv(".. /input/lanl-master-s-features-creating-0/train_X_features_865.csv")
 X_train_2 = pd.read_csv(".. /input/lanl-master-s-features-creating-1/train_X_features_865.csv")
 y_1 = pd.read_csv(".. /input/lanl-master-s-features-creating-0/train_y.csv", index_col=False, header=None)
 y_2 = pd.read_csv(".. /input/lanl-master-s-features-creating-1/train_y.csv", index_col=False, header=None )<concatenate>
if os.path.exists(data_enriched_path):
data = pd.read_hdf(data_enriched_path, 'data')
 else:
tmdb.API_KEY = 'YOUR_KEY_HERE'
search = tmdb.Search()
hits_cout = 0
attributes_to_fill = ['crew', 'cast', 'Keywords', 'genres', 'production_companies', 'production_countries', 'spoken_languages']
missing_credits_mask =(data[attributes_to_fill[0]].map(lambda x: len(x)== 0))
for col in attributes_to_fill[1:]:
missing_credits_mask = missing_credits_mask |(data[col].map(lambda x: len(x)== 0))
total_number_of_movies_to_fill = sum(missing_credits_mask)
 for col in attributes_to_fill:
data.loc[missing_credits_mask, col] = data.loc[missing_credits_mask, col].map(lambda x: [] if isinstance(x, dict)else x)
 for index, movie_row in tqdm(data[missing_credits_mask].iterrows() , total=total_number_of_movies_to_fill):
response = search.movie(query=movie_row['title'])if np.isnan(movie_row['release_year'])else search.movie(query=movie_row['title'], year=movie_row['release_year'])
 if response['total_results'] == 1:
movie = tmdb.Movies(response['results'][0]['id'])
 movie_info = movie.info(append_to_response='credits,keywords')
 if len(data.loc[index, 'cast'])== 0:
data.loc[index, 'cast'].extend(movie_info['credits']['cast'])
 if len(data.loc[index, 'crew'])== 0:
data.loc[index, 'crew'].extend(movie_info['credits']['crew'])
 if len(data.loc[index, 'Keywords'])== 0:
data.loc[index, 'Keywords'].extend(movie_info['keywords']['keywords'])
 for col in attributes_to_fill[3:]:
if len(data.loc[index, col])== 0:
data.loc[index, col].extend(movie_info[col])
 hits_cout += int(len(movie_info['credits']['cast'])> 0 or len(movie_info['credits']['crew'])> 0)
 print(f'Filling in was successful for {hits_cout} movies out of {total_number_of_movies_to_fill}.')
 data.to_hdf(data_enriched_path, 'data' )
X_train = pd.concat([X_train_1, X_train_2], axis=0)
 X_train = X_train.reset_index(drop=True)
 X_train.shape<concatenate>
data['genders_0_crew'] = data['crew'].apply(lambda x: sum([i['gender'] == 0 for i in x])if isinstance(x, list)else np.nan)
 data['genders_1_crew'] = data['crew'].apply(lambda x: sum([i['gender'] == 1 for i in x])if isinstance(x, list)else np.nan)
 data['genders_2_crew'] = data['crew'].apply(lambda x: sum([i['gender'] == 2 for i in x])if isinstance(x, list)else np.nan)
 data['_collection_name'] = data['belongs_to_collection'].apply(lambda x: x[0]['name'] if isinstance(x, list)else np.nan)
 le = LabelEncoder()
data['_collection_name'] = le.fit_transform(data['_collection_name'].map(str))
data.loc[data['_collection_name'].map(str)== 'nan', '_collection_name'] = np.nan
data['_num_Keywords'] = data['Keywords'].apply(lambda x: len(x)if isinstance(x, list)else np.nan)
 data['_num_cast'] = data['cast'].apply(lambda x: len(x)if isinstance(x, list)else np.nan)
 data['production_countries_count'] = data['production_countries'].apply(lambda x : len(x)if isinstance(x, list)else np.nan)
 data['production_companies_count'] = data['production_companies'].apply(lambda x : len(x)if isinstance(x, list)else np.nan)
 data['cast_count'] = data['cast'].apply(lambda x : len(x)if isinstance(x, list)else np.nan)
 data['crew_count'] = data['crew'].apply(lambda x : len(x)if isinstance(x, list)else np.nan )
y = pd.concat([y_1, y_2], axis=0)
 y = y.reset_index(drop=True)
 y.shape<load_from_csv>
%%HTML
<a id='revenue-inflation'></a>
X_test = pd.read_csv(".. /input/lanl-master-s-features-creating-0/test_X_features_10.csv" )<normalization>
dollar_costs_df = pd.DataFrame({'year': mean_yearly_revenues['release_year'], 'dollar_2018_cost': dollar_costs})
 data['dollar_2018_cost'] = data[['release_year']].merge(dollar_costs_df, left_on='release_year', right_on='year')['dollar_2018_cost']
scaler = StandardScaler()
train_columns = X_train.columns
X_train[train_columns] = scaler.fit_transform(X_train[train_columns])
 X_test[train_columns] = scaler.transform(X_test[train_columns] )<define_variables>
train.loc[(train['release_date'] >= '2015-01-01')&(train['revenue'] > 2e9), ['title']]
train_columns = X_train.columns
n_fold = 5<split>
%%HTML
<img src=https://media.giphy.com/media/xT0xejJnePNcOWoHOo/giphy.gif width=300>
%%time
folds = KFold(n_splits=n_fold, shuffle=True, random_state=42)
 oof = np.zeros(len(X_train))
train_score = []
cat_predictions = np.zeros(len(X_test))
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X_train, y_train.values)) :
print("fold {}".format(fold_))
X_tr, X_val = X_train[train_columns].iloc[trn_idx], X_train[train_columns].iloc[val_idx]
y_tr, y_val = y_train.iloc[trn_idx], y_train.iloc[val_idx]
model = CatBoostRegressor(n_estimators=25000, verbose=-1, objective="MAE", loss_function="MAE", boosting_type="Ordered", task_type="GPU")
 model.fit(X_tr, y_tr, eval_set=[(X_val, y_val)], verbose=2500, early_stopping_rounds=500)
 oof[val_idx] = model.predict(X_val)
 cat_predictions += model.predict(X_test[train_columns])/ folds.n_splits
train_score.append(model.best_score_['learn']["MAE"])
 cv_score = mean_absolute_error(y_train, oof)
 print(f"After {n_fold}: test_CV = {cv_score:.3f} | train_CV = {np.mean(train_score):.3f} | {cv_score-np.mean(train_score):.3f}", end=" " )<choose_model_class>
%%HTML
<img src=http://filmsplusmovies.com/wp-content/uploads/2014/06/317-hollywood.jpg width="500">
def create_model(input_dim=10):
model = keras.Sequential()
model.add(keras.layers.Dense(256, activation="relu", input_dim=input_dim))
model.add(keras.layers.Dropout(0.3))
model.add(keras.layers.Dense(128, activation="relu"))
model.add(keras.layers.Dropout(0.3))
model.add(keras.layers.Dense(96, activation="relu"))
model.add(keras.layers.Dropout(0.3))
model.add(keras.layers.Dense(1, activation="linear"))
optimizer = keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
 model.compile(optimizer=optimizer, loss='mae')
 return model
patience = 50
call_ES = keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=patience, verbose=1, mode='auto', baseline=None, restore_best_weights=True )<train_model>
data.drop('budget', axis=1, inplace=True)
 data['revenue'] = data['revenue'].map(np.log1p)
 data['_budget_runtime_ratio'] = data['budget_adjusted']/data['runtime']
data['_budget_popularity_ratio'] = data['budget_adjusted']/data['popularity_mean']
data['_budget_year_ratio'] = data['budget_adjusted']/(data['release_year']*data['release_year'])
 data['_budget_rating_ratio'] = data['budget_adjusted']/data['rating']
data['_budget_totalVotes_ratio'] = data['budget_adjusted']/data['totalVotes']
%%time
folds = KFold(n_splits=n_fold, shuffle=True, random_state=42)
 NN_oof = np.zeros(len(X_train))
train_score = []
NN_predictions = np.zeros(len(X_test))
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X_train, y_train.values)) :
print("fold {}".format(fold_))
X_tr, X_val = X_train[train_columns].iloc[trn_idx], X_train[train_columns].iloc[val_idx]
y_tr, y_val = y_train.iloc[trn_idx], y_train.iloc[val_idx]
model = create_model(X_train.shape[-1])
 model.fit(X_tr, y_tr, epochs=500, batch_size=32, verbose=0, callbacks=[call_ES,], validation_data=[X_val, y_val])
 NN_oof[val_idx] = model.predict(X_val)[:, 0]
NN_predictions += model.predict(X_test[train_columns])[:, 0] / folds.n_splits
history = model.history.history
tr_loss = history["loss"]
val_loss = history["val_loss"]
print(f"loss: {tr_loss[-patience]:.3f} | val_loss: {val_loss[-patience]:.3f} | diff: {val_loss[-patience]-tr_loss[-patience]:.3f}")
 train_score.append(tr_loss[-patience])
 cv_score = mean_absolute_error(y_train, NN_oof)
 print(f"After {n_fold}: test_CV = {cv_score:.3f} | train_CV = {np.mean(train_score):.3f} | {cv_score-np.mean(train_score):.3f}", end=" " )<load_from_csv>
jobs_of_interest_set = set(top_frequent_jobs.index)
 def value_counts_in_list(input_list):
counts = defaultdict(int)
 for val in input_list:
counts[val] += 1
return list(counts.items())
 job_person_counts = pd.Series(reduce(lambda list1, list2: list1 + list2,
data['crew'].map(lambda x: value_counts_in_list(
 filter(lambda x: x in jobs_of_interest_set,
map(lambda el: el['job'],
x)
)) ).values))
job_head_counts_df = pd.DataFrame({'job': job_person_counts.map(lambda x: x[0]), 'headcount': job_person_counts.map(lambda x: x[1])})
 percentile_95_job_headcount = job_head_counts_df.groupby('job')['headcount'].quantile(0.95 ).map(int)
 percentile_95_job_headcount[top_frequent_jobs.index]
Scirpus_predictions = pd.read_csv(".. /input/andrews-new-script-plus-a-genetic-program-model/gpI.csv")
 Scirpus_predictions.head()<save_to_csv>
columns_len_init = len(data.columns)
submission = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv')
 submission["time_to_failure"] =(cat_predictions + NN_predictions + Scirpus_predictions.time_to_failure.values)/ 3
submission.to_csv('submission.csv', index=False)
 submission.head()<import_modules>
print(data.isnull().sum().sort_values() [:1])
 count_col = data.isnull().sum().sort_values().index[0]
import matplotlib.pyplot as plt
from tqdm import tqdm
from sklearn.preprocessing import StandardScaler
from sklearn.svm import NuSVR
from sklearn.metrics import mean_absolute_error<load_from_csv>
def return_joint_entry(crew, job_name, sep='^'):
headcount = percentile_95_job_headcount[job_name]
if not isinstance(crew, list)and np.isnan(crew):
return np.nan
names = [el['name'] for el in crew if el['job'] == job_name][:headcount]
return sep.join(names)
 for crew_col in top_frequent_jobs.index:
data[crew_col] = data['crew'].map(lambda x: return_joint_entry(x, crew_col))
jobs_to_dummify = [job for job in top_frequent_jobs.index if percentile_95_job_headcount[job] > 2]
final_dummy_cols = []
def get_number_of_movies_with_name(pd_series, sep='^'):
names_in_movies = pd.Series(reduce(lambda list1, list2: list1 + list2,
pd_series.map(lambda x: x.split(sep)).values))
number_of_movies_with_name = names_in_movies.value_counts()
return number_of_movies_with_name
for crew_col in jobs_to_dummify:
number_of_movies_with_name = get_number_of_movies_with_name(data[crew_col])
 number_of_movies_with_name_train = get_number_of_movies_with_name(data.loc[data['dataset'] == 'Train', crew_col])
 remaining_names = set(number_of_movies_with_name.index[number_of_movies_with_name >= 10] ).intersection(
 set(number_of_movies_with_name_train.index[number_of_movies_with_name_train >= 5]))
df_added_cols = data[crew_col].map(lambda x: '^'.join([name for name in x.split('^')if name in remaining_names])).str.get_dummies(sep='^')
 df_added_cols.columns = [f'{crew_col}_{value}' for value in df_added_cols.columns]
final_dummy_cols.extend(df_added_cols.columns)
 data = pd.concat(( data, df_added_cols), axis=1)
 data.drop(jobs_to_dummify + ['crew'], axis=1, inplace=True)
 jobs_joint = [job for job in top_frequent_jobs.index if percentile_95_job_headcount[job] <= 2]
def hide_rare_values(data, column, overall_limit=7, train_limit=3):
counts_total = data.groupby(joint_col)[count_col].count()
counts_train = data[data['dataset']=='Train'].groupby(joint_col)[count_col].count()
names_to_forget = set(counts_total[counts_total < overall_limit].index ).union(set(counts_train[counts_train < train_limit].index))
data.loc[data[joint_col].isin(names_to_forget), joint_col] = np.nan
return data
for joint_col in jobs_joint:
data = hide_rare_values(data, joint_col )
train = pd.read_csv('.. /input/train.csv',nrows = 10**5, dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64})
<set_options>
def return_entries(list_dicts, sep='^'):
if not isinstance(list_dicts, list)and np.isnan(list_dicts):
return np.nan
entries = [el['name'] for el in list_dicts]
return sep.join(entries)
 data['genres'] = data['genres'].map(return_entries)
 data = hide_rare_values(data, 'genres', overall_limit=50, train_limit=20)
 for dict_col in ['cast', 'Keywords', 'production_companies', 'production_countries', 'spoken_languages']:
data[dict_col] = data[dict_col].map(return_entries)
 names_in_movies = pd.Series(reduce(lambda list1, list2: list1 + list2,
data[dict_col].map(lambda x: x.split('^')).values))
number_of_movies_with_name = names_in_movies.value_counts()
min_movie_count = 20 if dict_col == 'cast' else 100
remaining_names = number_of_movies_with_name.index[number_of_movies_with_name >= min_movie_count]
df_added_cols = data[dict_col].map(lambda x: '^'.join([name for name in x.split('^')if name in remaining_names])).str.get_dummies(sep='^')
 df_added_cols.columns = [f'{dict_col}_{value}' for value in df_added_cols.columns]
final_dummy_cols.extend(df_added_cols.columns)
 data = pd.concat(( data, df_added_cols), axis=1)
 data.drop(['cast', 'Keywords', 'production_companies', 'production_countries', 'spoken_languages'], axis=1, inplace=True )
pd.options.display.precision = 15<rename_columns>
porter = PorterStemmer()
def get_listofwords_series(col):
all_list_of_words_series = data[col].map(str ).map(lambda text: [porter.stem(w)if w != 'nan' else np.nan for w in word_tokenize(text.translate(str.maketrans('','',string.punctuation)).lower())])
 if f'{col}_stopwords_count' not in data.columns:
data[f'{col}_stopwords_count'] = all_list_of_words_series.map(lambda x: sum([w in stopwords_eng for w in x]))
data[f'{col}_stopwords_fraction'] = data[f'{col}_stopwords_count']/all_list_of_words_series.map(len)
 return all_list_of_words_series.map(lambda all_words: list(filter(lambda x: x not in stopwords_eng, all_words)) )
train.rename({"acoustic_data": "signal", "time_to_failure": "time"}, axis="columns", inplace=True )<data_type_conversions>
%%HTML
<iframe width="560" height="315" src="https://www.youtube.com/embed/_wYtG7aQTHA?rel=0&amp;controls=1&amp;showinfo=0" frameborder="0" allowfullscreen></iframe>
train = train.fillna(train.mean() )<feature_engineering>
data = add_word_columns(data, 'overview' )
<data_type_conversions>
%%HTML
<img src=https://mindblown.smumn.edu/wp-content/uploads/2018/02/zipf-law-meme.jpg width="200">
x_train=x_train.fillna(x_train.mean() )<create_dataframe>
data = add_word_columns(data, 'tagline' )
<prepare_x_and_y>
data.drop(['title', 'original_title', 'overview', 'tagline'], axis=1, inplace=True )
<prepare_x_and_y>
def compute_stats(val_date, competition_values_all):
movie_val = val_date[0]
movie_competition_groupy_id_date = val_date[1]
if np.isnan(movie_val)or pd.isnull(movie_competition_groupy_id_date):
return [np.nan] * 6
competition = competition_values_all[movie_competition_groupy_id_date]
zscore =(movie_val - np.nanmean(competition)) /np.nanstd(competition)
 percentile = stats.percentileofscore(competition, movie_val)
 val_to_max = movie_val / np.nanmax(competition)
 min_to_val = np.nanmin(competition)/ movie_val
val_to_mean = movie_val / np.nanmean(competition)
 val_to_median = movie_val / np.nanmedian(competition)
 return [zscore, percentile, val_to_max, min_to_val, val_to_mean, val_to_median]
def add_comparison_stats(data, cols, neighbourhood_in_weeks=1):
def get_next_sunday_date(date):
return date + pd.to_timedelta(6 - date.weekday() % 7, unit='D')
 time_range_groups = None
dummy_release_dates = []
for i in range(2*neighbourhood_in_weeks):
ts_dummy = data.loc[~data['release_date'].isin(dummy_release_dates), 'release_date'].min() - pd.to_timedelta(i * 1, unit='W')
 dummy_row = pd.DataFrame([np.full_like(data.iloc[0], fill_value=np.nan)], columns=data.columns)
 dummy_row['release_date'] = ts_dummy
if i > 0:
dummy_release_dates.append(ts_dummy)
 data = pd.concat(( data, dummy_row))
groups_next =(data[['release_date'] + cols] 
.groupby(pd.Grouper(key='release_date', freq=f'{2*neighbourhood_in_weeks}W', closed='right')) 
.agg(lambda values: list(filter(lambda val: not np.isnan(val), values))))
time_range_groups = groups_next if time_range_groups is None else pd.concat(( time_range_groups, groups_next))
for col in cols:
competition_values = time_range_groups[col]
val_date_series = data[col].map(lambda x: [x])+(data['release_date'] 
.map(lambda x: x + pd.to_timedelta(neighbourhood_in_weeks, unit='W')) 
.map(get_next_sunday_date)
.map(lambda x: [x]))
column_names = [f"{col}_competition_{stat}_±_{neighbourhood_in_weeks}_week{'s' if neighbourhood_in_weeks > 1 else ''}" for stat in ['zscore', 'percentile', 'val_to_max', 'min_to_val', 'val_to_mean', 'val_to_median']]
df_added_cols = pd.DataFrame(np.column_stack(val_date_series.map(lambda val_date: compute_stats(val_date, competition_values)) ).T, columns=column_names, index=data.index)
 data = pd.concat(( data, df_added_cols), axis=1)
 return data
x_train = train[['signal']]
y_train = train[['time']]<normalization>
data = add_comparison_stats(data, cols=['rating', 'popularity_mean'])
scaler = StandardScaler()
scaler.fit(x_train)
 X_train_scaled = scaler.transform(x_train )<train_on_grid>
data.drop(['belongs_to_collection', 'popularity', 'popularity2', 'popularity3'], axis=1, inplace=True)
 data['collection_id'] = data['collection_id'].astype('object' )
svm = NuSVR()
svm.fit(X_train_scaled, y_train.values.flatten())
 y_pred = svm.predict(X_train_scaled )<compute_test_metric>
data.drop(['homepage', 'id', 'imdb_id', 'poster_path', 'release_date', 'budget_cat'], axis=1, inplace=True )
score = mean_absolute_error(y_train.values.flatten() , y_pred)
 print(f'Score: {score:0.3f}' )<load_from_csv>
%%HTML
<iframe width="560" height="315" src="https://www.youtube.com/embed/1cRGpDXTJC8?rel=0&amp;controls=1&amp;showinfo=0" frameborder="0" allowfullscreen></iframe>
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 submission.head()<create_dataframe>
categorical_columns = []
numerical_columns = []
for col, col_type in zip(data.columns, data.dtypes):
if col == 'dataset':
continue
if col_type==np.dtype('O')or col in final_dummy_cols:
categorical_columns.append(col)
 else:
numerical_columns.append(col )
X_test = pd.DataFrame(columns=x_train.columns, dtype=np.float64, index=submission.index )<load_from_csv>
flag_cols = ['has_homepage',
'isbelongs_to_collectionNA',
'isTaglineNA',
'isOriginalLanguageEng',
'isTitleDifferent',
'isMovieReleased']
numerical_columns = [col for col in numerical_columns if col not in flag_cols]
categorical_columns.extend(flag_cols)
 numerical_columns.remove('revenue' )
for seg_id in X_test.index:
seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 x = seg['acoustic_data'].values
print(x)
 X_test.loc[seg_id,'signal'] = np.median(x )<count_missing_values>
data_ffm = deepcopy(data)
 for col in numerical_columns:
if col != 'release_year':
data_ffm[col] = pd.qcut(data_ffm[col], q=200, duplicates='drop', labels=False ).factorize() [0] + 1
for col in categorical_columns:
data_ffm[col] = data_ffm[col].factorize() [0] + 1
train_logical_indexing = data_ffm['dataset']=='Train'
test_logical_indexing = data_ffm['dataset']=='Test'
for col in [num_col for num_col in numerical_columns if num_col != 'release_year'] + [cat_col for cat_col in categorical_columns if cat_col != 'dataset']:
train_values = set(data_ffm.loc[train_logical_indexing, col].values)
 test_values = set(data_ffm.loc[test_logical_indexing, col].values)
 values_to_preserve = train_values.intersection(test_values)
 data_ffm.loc[~data_ffm[col].isin(values_to_preserve), col] = 0
X_test.isnull().sum()<save_to_csv>
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 cv_folds = [[trn_idx, val_idx] for trn_idx, val_idx in skf.split(list(range(( data_ffm['dataset'] == 'Train' ).sum())) ,
data_ffm.loc[data_ffm['dataset'] == 'Train', 'release_year'])]
data_ffm['release_year'] = data_ffm['release_year'].factorize() [0]
train_ffm = data_ffm[data_ffm['dataset'] == 'Train'].drop('dataset', axis=1)
 test_ffm = data_ffm[data_ffm['dataset'] == 'Test'].drop('dataset', axis=1 )
X_test_scaled = scaler.transform(X_test)
 submission['time_to_failure'] = svm.predict(X_test_scaled)
 submission.to_csv('submission.csv' )<import_modules>
accumulated_count = 0
dict_feature = dict()
for col in numerical_columns + categorical_columns:
for val in data_ffm[col].unique() :
dict_feature[(col, val)] = accumulated_count + val
accumulated_count += data_ffm[col].nunique()
def produce_file(file_name, df_to_process, test=False):
noofrows = df_to_process.shape[0]
noofcolumns = df_to_process.shape[1]
currentcode = len(numerical_columns)
 with open(file_name, 'w')as text_file:
for n, r in enumerate(tqdm(range(noofrows), desc=file_name)) :
datastring = ''
datarow = df_to_process.iloc[r]
if not test:
datastring += str(datarow['revenue'])
 else:
datastring += '10'
for col_i, col in enumerate(numerical_columns + categorical_columns):
datastring = f'{datastring} {col_i}:{dict_feature[(col, datarow[col])]}:1'
datastring += '
'
text_file.write(datastring)
 def create_train_val_files(train_ffm, val_ffm, test_ffm=test_ffm, fold_idx=0, dict_feature=dict_feature):
produce_file(f'train_ffm_{fold_idx}', train_ffm)
 produce_file(f'val_ffm_{fold_idx}', val_ffm)
 produce_file('test_ffm', test_ffm, test=True )
warnings.filterwarnings("ignore")
 print(os.listdir(".. /input"))<load_from_csv>
submission_ffm = pd.read_csv('.. /input/tmdb-box-office-prediction/sample_submission.csv')
 submission_ffm['revenue'] = 0
cv_rmsle_scores = []
for fold_idx,(trn_idx, val_idx)in enumerate(cv_folds):
print('='*40)
 print('='*16 + f' FOLD {fold_idx} ' + '='*16)
 train_ffm_trn = train_ffm.iloc[trn_idx]
train_ffm_val = train_ffm.iloc[val_idx]
create_train_val_files(train_ffm_trn, train_ffm_val, fold_idx=fold_idx)
 val_score = execute(f'libffm-regression/ffm-train -p val_ffm_{fold_idx} --auto-stop -t 1 train_ffm_{fold_idx} model_{fold_idx}', parse_val_result=True)
 print(f'Val score: {val_score}')
 cv_rmsle_scores.append(val_score)
 os.remove(f'val_ffm_{fold_idx}')
 os.remove(f'train_ffm_{fold_idx}')
 execute(f'libffm-regression/ffm-predict test_ffm model_{fold_idx} output_{fold_idx} > /dev/null')
 predictions_current = pd.read_csv(f'output_{fold_idx}',header=None)
 os.remove(f'output_{fold_idx}')
 os.remove(f'model_{fold_idx}')
 predictions_current.columns = ['revenue']
submission_ffm['revenue'] += predictions_current['revenue']/len(cv_folds)
 rmtree('libffm-regression')
 os.remove('test_ffm')
 rmsle_mean = np.mean(cv_rmsle_scores)
 rmsle_mean_std = np.std(cv_rmsle_scores, ddof=1)/np.sqrt(len(cv_rmsle_scores))
tinteval_lb, tinterval_ub = _tconfint_generic(rmsle_mean, rmsle_mean_std, dof=len(cv_rmsle_scores)-1, alpha=0.05, alternative='two-sided')
 print(f"95% confidence interval for RMSLE is:({tinteval_lb:.2f}, {tinterval_ub:.2f} ).")
 submission_ffm['revenue'] = submission_ffm['revenue'].map(np.expm1)
train_X = pd.read_csv('.. /input/earthquake-data-overlapping4/train_X.csv')
 train_y = pd.read_csv('.. /input/earthquake-data-overlapping4/train_y.csv')
 test_X = pd.read_csv('.. /input/earthquake-data-overlapping4/test_X.csv')
 train_X.head()<normalization>
%%HTML
<a id=lgb></a>
scaler = StandardScaler()
scaler.fit(train_X)
 scaled_train_X = pd.DataFrame(scaler.transform(train_X), columns=train_X.columns)
<create_dataframe>
%%HTML
<iframe width="560" height="315" src="https://www.youtube.com/embed/mg7BNAvHeII?rel=0&amp;controls=1&amp;showinfo=0" frameborder="0" allowfullscreen></iframe>
scaled_test_X = pd.DataFrame(scaler.transform(test_X), columns=test_X.columns)
<feature_engineering>
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 train_data = data[data['dataset'] == 'Train']
y_train = train_data['revenue']
cv_folds = [[trn_idx, val_idx] for trn_idx, val_idx in skf.split(list(range(len(train_data))), train_data['release_year'])]
def hide_rare_vals(data, categorical_columns, min_count=2):
for col in categorical_columns:
data['count'] = np.arange(len(data))
value_counts = data.groupby(col, as_index=False)['count'].count()
values_to_discard = set(value_counts.loc[value_counts['count'] < min_count, col].values)
 data.loc[data[col].isin(values_to_discard), col] = np.nan
data.drop('count', axis=1, inplace=True)
 return data
def get_train_lgb(data):
categorical_columns = [col for col, col_type in zip(data.columns, data.dtypes)if col_type==np.dtype('O')and col != 'dataset']
data_for_boosting = hide_rare_vals(data,
[col for col in categorical_columns if 'word' in col],
min_count=10)
 data_one_hot_enc = pd.get_dummies(data_for_boosting.drop(['revenue'], axis=1),
columns=categorical_columns)
 train_one_hot_enc = data_one_hot_enc[data_one_hot_enc['dataset'] == 'Train'].drop(['dataset'], axis=1)
 return train_one_hot_enc, data_one_hot_enc
train_one_hot_enc, data_one_hot_enc = get_train_lgb(data )
train_X_mean = np.mean(scaled_train_X, axis=0)
 train_X_mean.head()<feature_engineering>
counter = 0
def onstep(res):
global counter
x0 = res.x_iters
y0 = res.func_vals
print('Current iter: ', counter,
' - Score ', res.fun,
' - Args: ', res.x)
 joblib.dump(( x0, y0), 'checkpoint.pkl')
 counter += 1
def make_objective(model, dimensions, df=data):
model = deepcopy(model)
 @use_named_args(dimensions)
 def objective(**params):
model.set_params(**params)
 score = 0
for trn_idx, val_idx in cv_folds:
X_fit, X_val = df.iloc[trn_idx], df.iloc[val_idx]
y_fit, y_val = y_train.iloc[trn_idx], y_train.iloc[val_idx]
model.fit(X_fit, y_fit,
eval_set=[(X_val, y_val)],
verbose=False, early_stopping_rounds=30)
 score += np.sqrt(mean_squared_error(y_val, model.predict(X_val)))/len(cv_folds)
 return score
return objective
test_X_mean = np.mean(scaled_test_X, axis=0)
 test_X_mean.head()<sort_values>
skopt_params = {'learning_rate': 0.005, 'num_leaves': 2558, 'max_depth': 0, 'min_child_samples': 15, 'max_bin': 95637,
'subsample': 0.4462883478742228, 'subsample_freq': 0, 'colsample_bytree': 0.12636976300879604, 'min_child_weight': 1, 'subsample_for_bin': 422967, 'reg_lambda': 8.991454050632966e-05, 'reg_alpha': 0.000956883899855844, 'scale_pos_weight': 0.000511057873147196}
def get_skopt_params_lgb(train_one_hot_enc):
clf = LGBMRegressor(boosting_type='gbdt',
objective='rmse',
n_jobs=-1,
verbose=0,
n_estimators=10000)
 dimensions = [Real(0.01, 1.0, 'log-uniform', name='learning_rate'),
Integer(50, 1000, name='num_leaves'),
Integer(0, 500, name='max_depth'),
Integer(0, 200, name='min_child_samples'),
Integer(100, 100000, name='max_bin'),
Real(0.01, 1.0, 'uniform', name='subsample'),
Integer(0, 10, name='subsample_freq'),
Real(0.01, 1.0, 'uniform', name='colsample_bytree'),
Integer(0, 10, name='min_child_weight'),
Integer(100000, 500000, name='subsample_for_bin'),
Real(1e-9, 1000, 'log-uniform', name='reg_lambda'),
Real(1e-9, 1.0, 'log-uniform', name='reg_alpha'),
Real(1e-6, 500, 'log-uniform', name='scale_pos_weight')]
objective = make_objective(clf, dimensions, train_one_hot_enc)
 gp_round = gp_minimize(func=objective,
dimensions=dimensions,
acq_func='gp_hedge',
n_calls=25,
callback=[onstep],
random_state=42)
 best_parameters = gp_round.x
best_result = gp_round.fun
skopt_params = {param.name: value for param, value in zip(dimensions, best_parameters)}
print(f'Best RMSLE: {best_result},
corresponding parameters: {skopt_params}')
 return skopt_params
if skopt_params is None:
skopt_params = get_skopt_params_lgb(train_one_hot_enc )
c = 0
for _, i in(train_X_mean - test_X_mean ).abs().sort_values().iteritems() :
print(i, _)
 if _ == 'std_roll_std_50':
print(c - 9)
 c += 1<sort_values>
def train_lgb(train_one_hot_enc):
clfs = []
cv_rmsle_scores = []
importances = pd.DataFrame()
for fold_idx,(trn_idx, val_idx)in enumerate(cv_folds):
clf = LGBMRegressor(boosting_type='gbdt',
objective='rmse',
num_leaves=5000,
n_jobs=-1,
verbose=1,
n_estimators=15000,
min_data=50)
 clf.set_params(**skopt_params)
 X_fit, X_val = train_one_hot_enc.iloc[trn_idx], train_one_hot_enc.iloc[val_idx]
y_fit, y_val = y_train.iloc[trn_idx], y_train.iloc[val_idx]
clf.fit(X_fit, y_fit, eval_metric='rmse',
eval_set=[(X_val, y_val)],
verbose=500, early_stopping_rounds=300)
 imp_df = pd.DataFrame({
'feature': X_fit.columns,
'gain': clf.feature_importances_,
'fold': [fold_idx + 1] * len(X_fit.columns),
})
 importances = pd.concat([importances, imp_df], axis=0, sort=False)
 clfs.append(clf)
 cv_rmsle_scores.append(np.sqrt(mean_squared_error(y_val, clf.predict(X_val))))
rmsle_mean = np.mean(cv_rmsle_scores)
 rmsle_mean_std = np.std(cv_rmsle_scores, ddof=1)/np.sqrt(len(cv_rmsle_scores))
tinteval_lb, tinterval_ub = _tconfint_generic(rmsle_mean, rmsle_mean_std, dof=len(cv_rmsle_scores)-1, alpha=0.05, alternative='two-sided')
 print(f"95% confidence interval for RMSLE is:({tinteval_lb:.2f}, {tinterval_ub:.2f} ).")
 return clfs, importances
clfs, importances = train_lgb(train_one_hot_enc )
features =(train_X_mean - test_X_mean ).abs().sort_values()
features = features[features > 0]
drop_features = features.index[167:]<normalization>
%%HTML
<a id=catboost></a>
train_X_less = scaled_train_X.drop(drop_features, axis=1)
 test_X_less = scaled_test_X.drop(drop_features, axis=1 )<compute_train_metric>
def get_data_cat(data):
data_catboost = data.drop('revenue', axis=1)
 train_logical_idx = data_catboost['dataset'] == 'Train'
test_logical_idx = data_catboost['dataset'] == 'Test'
data_catboost.drop('dataset', axis=1, inplace=True)
 init_cat_features = {'has_homepage',
'isbelongs_to_collectionNA',
'isTaglineNA',
'isOriginalLanguageEng',
'isTitleDifferent',
'isMovieReleased'}
cat_features=[i for i,(col, col_type)in enumerate(zip(data_catboost.columns, data_catboost.dtypes))
if col_type==np.dtype('O')or col in init_cat_features]
for col in cat_features:
data_catboost.iloc[:, col] = data_catboost.iloc[:, col].map(str)
 train_cat = data_catboost[train_logical_idx]
test_cat = data_catboost[test_logical_idx]
return train_cat, test_cat, cat_features
train_data_cat, test_data_cat, cat_features = get_data_cat(data)
 skopt_params_catboost = {'learning_rate': 0.02, 'depth': 4, 'random_strength': 2.6832935037024834e-08, 'bagging_temperature': 0.650888472948853, 'border_count': 15, 'l2_leaf_reg': 22}
def get_skopt_params_cat(train_data_cat):
clf = CatBoostRegressor(loss_function='RMSE',
verbose=False,
cat_features=cat_features,
thread_count=12)
 dimensions = [Real(0.01, 1.0, 'log-uniform', name='learning_rate'),
Integer(1, 10, name='depth'),
Real(1e-9, 10, 'log-uniform', name='random_strength'),
Real(0.0, 1.0, name='bagging_temperature'),
Integer(1, 255, name='border_count'),
Integer(2, 30, name='l2_leaf_reg')]
objective = make_objective(clf, dimensions, train_data_cat)
 gp_round = gp_minimize(func=objective,
dimensions=dimensions,
acq_func='gp_hedge',
n_calls=15,
callback=[onstep],
random_state=42)
 best_parameters = gp_round.x
best_result = gp_round.fun
skopt_params_catboost = {param.name: value for param, value in zip(dimensions, best_parameters)}
print(f'Best RMSLE: {best_result},
corresponding parameters: {skopt_params_catboost}')
 return skopt_params_catboost
if skopt_params_catboost is None:
skopt_params_catboost = get_skopt_params_cat(train_data_cat )
n_fold = 5
def mae_cv(model):
folds = KFold(n_splits=n_fold, shuffle=True, random_state=42 ).get_n_splits(train_X_less.values)
 mae = -cross_val_score(model, train_X_less.values, train_y, scoring="neg_mean_absolute_error",
verbose=0,
cv=folds)
 return mae<compute_train_metric>
def train_catboost(train_data_cat, cat_features):
clfs_cat = []
cv_rmsle_scores = []
importances_cat = pd.DataFrame()
for fold_idx,(trn_idx, val_idx)in enumerate(cv_folds):
clf = CatBoostRegressor(loss_function='RMSE',
verbose=False,
cat_features=cat_features,
thread_count=12,
iterations=15000)
 clf.set_params(**skopt_params_catboost)
 X_fit, X_val = train_data_cat.iloc[trn_idx], train_data_cat.iloc[val_idx]
y_fit, y_val = y_train.iloc[trn_idx], y_train.iloc[val_idx]
clf.fit(X_fit, y_fit,
eval_set=[(X_fit, y_fit),(X_val, y_val)],
verbose=100, early_stopping_rounds=50)
 imp_df_cat = pd.DataFrame({
'feature': X_fit.columns,
'gain': clf.feature_importances_,
'fold': [fold_idx + 1] * len(X_fit.columns),
})
 importances_cat = pd.concat([importances_cat, imp_df_cat], axis=0, sort=False)
 clfs_cat.append(clf)
 cv_rmsle_scores.append(np.sqrt(mean_squared_error(y_val, clf.predict(X_val))))
rmsle_mean = np.mean(cv_rmsle_scores)
 rmsle_mean_std = np.std(cv_rmsle_scores, ddof=1)/np.sqrt(len(cv_rmsle_scores))
tinteval_lb, tinterval_ub = _tconfint_generic(rmsle_mean, rmsle_mean_std, dof=len(cv_rmsle_scores)-1, alpha=0.05, alternative='two-sided')
 print(f"95% confidence interval for RMSLE is:({tinteval_lb:.2f}, {tinterval_ub:.2f} ).")
 return clfs_cat, importances_cat
clfs_cat, importances_cat = train_catboost(train_data_cat, cat_features )
<train_model><EOS> 
test_one_hot_enc = data_one_hot_enc[data_one_hot_enc['dataset'] == 'Test'].drop(['dataset'], axis=1)
 submission_lgb = pd.read_csv('.. /input/tmdb-box-office-prediction/sample_submission.csv')
 submission_lgb['revenue'] = 0
for clf in clfs:
submission_lgb['revenue'] += clf.predict(test_one_hot_enc)/len(clfs)
 submission_cat = pd.read_csv('.. /input/tmdb-box-office-prediction/sample_submission.csv')
 submission_cat['revenue'] = 0
for clf in clfs_cat:
submission_cat['revenue'] += clf.predict(test_data_cat)/len(clfs_cat)
 cat_weight = 3
lgb_weight = 1
submission_lgb_cat = pd.read_csv('.. /input/tmdb-box-office-prediction/sample_submission.csv')
 submission_lgb_cat['revenue'] =(cat_weight*submission_cat['revenue'] + lgb_weight*submission_lgb['revenue'])/(cat_weight + lgb_weight)
 submission_lgb_cat['revenue'] = submission_lgb_cat['revenue'].map(np.expm1)
 submission_lgb_cat.to_csv('submission_lgb_cat.csv', index=None )