Kaggle competition: TMDB Box Office Prediction
gc.enable()
del credit, credit_stats_by_prev
gc.collect()<load_from_csv>
train = pd.read_csv('/kaggle/input/tmdb-box-office-prediction/train.csv')
 train.info()
install = pd.read_csv('.. /input/installments_payments.csv')
 install.head()<feature_engineering>
test = pd.read_csv('/kaggle/input/tmdb-box-office-prediction/test.csv')
 test.info()
install['DAYS_DIFF'] = install['DAYS_INSTALMENT'] - install['DAYS_ENTRY_PAYMENT']
install['AMT_DIFF'] = install['AMT_INSTALMENT'] - install['AMT_PAYMENT']
install.head()<drop_column>
import seaborn as sns
import xgboost as xgb
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import mean_squared_error
import plotly as py
import plotly.graph_objs as go
import matplotlib.pyplot as plt
from collections import Counter
install_stats_by_prev = install[['SK_ID_PREV', 'SK_ID_CURR']]<merge>
train.isna().values.sum()
install_NUM_INSTALMENT_VERSION_count = install.groupby('SK_ID_PREV', as_index=False)['NUM_INSTALMENT_VERSION'].count().rename(columns = {'NUM_INSTALMENT_VERSION': 'install_NUM_INSTALMENT_VERSION_count'})
 install_NUM_INSTALMENT_VERSION_max = install.groupby('SK_ID_PREV', as_index=False)['NUM_INSTALMENT_VERSION'].max().rename(columns = {'NUM_INSTALMENT_VERSION': 'install_NUM_INSTALMENT_VERSION_max'})
 install_stats_by_prev = install_stats_by_prev.merge(install_NUM_INSTALMENT_VERSION_count, on = 'SK_ID_PREV', how = 'left')
 install_stats_by_prev = install_stats_by_prev.merge(install_NUM_INSTALMENT_VERSION_max, on = 'SK_ID_PREV', how = 'left' )<set_options>
def get_name(data):
try:
x = eval(data)
 return x[0]['name']
except:
return ''
def get_len(data):
try:
x = eval(data)
 return len(x)
 except:
return 0
def all_data(data, key = 'name'):
try:
x = eval(data)
 return ' '.join(sorted([i[key] for i in x]))
except:
return
def make_list(data, key = 'name'):
try:
data = eval(data)
 return [i[key] for i in data]
except:
return []
def apply_encode(data, name):
try:
if name in data:
return 1
else:
return 0
except:
return
def get_gender(data,index):
try:
data = eval(str(data))
count = 0
for i in data:
if i['gender'] == index:
count += 1
return count
except:
return 0
def singular_verb(feature_name):
if feature_name == 'production_countries':
return 'production_country'
elif feature_name == 'production_companies':
return 'production_company'
elif feature_name == 'spoken_languages':
return 'spoken_language'
elif feature_name == 'Keywords':
return 'Keyword'
else:
return feature_name
def fix_date(x):
year = x.split('/')[2]
if int(year)<= 19:
return x[:-2] + '20' + year
else:
return x[:-2] + '19' + year
def process_date(df):
date_parts = ["year", "weekday", "month", 'weekofyear', 'day', 'quarter']
for part in date_parts:
part_col = 'release_date' + "_" + part
df[part_col] = getattr(df['release_date'].dt, part ).astype(int)
 return df
gc.enable()
del install_NUM_INSTALMENT_VERSION_count, install_NUM_INSTALMENT_VERSION_max
gc.collect()<merge>
train['runtime'].fillna(0, inplace=True)
 train['status'].fillna('Released', inplace=True)
 train['release_date'].fillna(train['release_date'].mode() [0], inplace=True)
 test['runtime'].fillna(0, inplace=True)
 test['status'].fillna('Released', inplace=True)
 test['release_date'].fillna(test['release_date'].mode() [0], inplace=True )
install_DAYS_INSTALMENT_mean = install.groupby('SK_ID_PREV', as_index=False)['DAYS_INSTALMENT'].mean().rename(columns = {'DAYS_INSTALMENT': 'install_DAYS_INSTALMENT_mean'})
 install_stats_by_prev = install_stats_by_prev.merge(install_DAYS_INSTALMENT_mean, on = 'SK_ID_PREV', how = 'left')
 gc.enable()
del install_DAYS_INSTALMENT_mean
gc.collect()<merge>
train['collection_name'] = train['belongs_to_collection'].apply(get_name)
 train['num_of_collection'] = train['belongs_to_collection'].apply(get_len)
 train['num_of_genres'] = train['genres'].apply(get_len)
 train['num_of_countries'] = train['production_countries'].apply(get_len)
 train['num_of_companies'] = train['production_companies'].apply(get_len)
 train['num_of_spoken_languages'] = train['spoken_languages'].apply(get_len)
 train['num_of_cast'] = train['cast'].apply(get_len)
 train['num_of_crew'] = train['crew'].apply(get_len)
 train['num_of_keywords'] = train['Keywords'].apply(get_len)
 train['has_homepage'] = 0
train.loc[train['homepage'].isnull() == False, 'has_homepage'] = 1
test['collection_name'] = test['belongs_to_collection'].apply(get_name)
 test['num_of_collection'] = test['belongs_to_collection'].apply(get_len)
 test['num_of_genres'] = test['genres'].apply(get_len)
 test['num_of_countries'] = test['production_countries'].apply(get_len)
 test['num_of_companies'] = test['production_companies'].apply(get_len)
 test['num_of_spoken_languages'] = test['spoken_languages'].apply(get_len)
 test['num_of_cast'] = test['cast'].apply(get_len)
 test['num_of_crew'] = test['crew'].apply(get_len)
 test['num_of_keywords'] = test['Keywords'].apply(get_len)
 test['has_homepage'] = 0
test.loc[test['homepage'].isnull() == False, 'has_homepage'] = 1
install_DAYS_ENTRY_PAYMENT_mean = install.groupby('SK_ID_PREV', as_index=False)['DAYS_ENTRY_PAYMENT'].mean().rename(columns = {'DAYS_ENTRY_PAYMENT': 'install_DAYS_ENTRY_PAYMENT_mean'})
 install_stats_by_prev = install_stats_by_prev.merge(install_DAYS_ENTRY_PAYMENT_mean, on = 'SK_ID_PREV', how = 'left')
 gc.enable()
del install_DAYS_ENTRY_PAYMENT_mean
gc.collect()<merge>
feature_list = ['genres', 'production_countries', 'production_companies',
'spoken_languages', 'Keywords']
for feature_name in feature_list:
field = train[feature_name].apply(all_data)
 list_feature = train[feature_name].apply(make_list)
 list_of_feature = list(list_feature.values)
 top_feature_data = [c[0] for c in Counter([i for j in list_of_feature for i in j] ).most_common(12)]
feature_name_sv = singular_verb(feature_name)
 for name in top_feature_data:
train[feature_name_sv + '_' + name] = field.apply(lambda x:apply_encode(x,name))
train[feature_name_sv + '_' + name] = train[feature_name_sv + '_' + name].fillna(0 ).astype('int32')
 train = train.drop([feature_name], axis=1)
 for feature_name in feature_list:
field = test[feature_name].apply(all_data)
 list_feature = test[feature_name].apply(make_list)
 list_of_feature = list(list_feature.values)
 top_feature_data = [c[0] for c in Counter([i for j in list_of_feature for i in j] ).most_common(12)]
feature_name_sv = singular_verb(feature_name)
 for name in top_feature_data:
test[feature_name_sv + '_' + name] = field.apply(lambda x:apply_encode(x,name))
test[feature_name_sv + '_' + name] = test[feature_name_sv + '_' + name].fillna(0 ).astype('int32')
 test = test.drop([feature_name], axis=1 )
install_AMT_INSTALMENT_mean = install.groupby('SK_ID_PREV', as_index=False)['AMT_INSTALMENT'].mean().rename(columns = {'AMT_INSTALMENT': 'install_AMT_INSTALMENT_mean'})
 install_stats_by_prev = install_stats_by_prev.merge(install_AMT_INSTALMENT_mean, on = 'SK_ID_PREV', how = 'left')
 gc.enable()
del install_AMT_INSTALMENT_mean
gc.collect()<merge>
feature_list = ['cast','crew']
for feature_name in feature_list:
if feature_name == 'cast':
list_keys = ['name','character']
for i_key in list_keys:
feature = train[feature_name].apply(lambda x: all_data(x,i_key))
list_feature = train[feature_name].apply(lambda x: make_list(x,i_key))
list_of_feature = list(list_feature.values)
 top_feature_data = [c[0] for c in Counter([i for j in list_of_feature for i in j] ).most_common(10)]
feature_name_sv = singular_verb(feature_name)
 for name in top_feature_data:
train[feature_name_sv + '_'+ i_key + '_' + name] = feature.apply(lambda x:apply_encode(x,name))
train[feature_name_sv + '_'+ i_key + '_' + name] = train[feature_name_sv + '_'+ i_key + '_' + name].fillna(0 ).astype('int32')
 del feature
del list_feature
if feature_name == 'crew':
list_keys = ['name','job','department']
for i_key in list_keys:
feature = train[feature_name].apply(lambda x: all_data(x,i_key))
list_feature = train[feature_name].apply(lambda x: make_list(x,i_key))
list_of_feature = list(list_feature.values)
 top_feature_data = [c[0] for c in Counter([i for j in list_of_feature for i in j] ).most_common(10)]
feature_name_sv = singular_verb(feature_name)
 for name in top_feature_data:
train[feature_name_sv + '_'+ i_key + '_' + name] = feature.apply(lambda x:apply_encode(x,name))
train[feature_name_sv + '_'+ i_key + '_' + name] = train[feature_name_sv + '_'+ i_key + '_' + name].fillna(0 ).astype('int32')
 del feature
del list_feature
train[feature_name +'_genders_0'] = train[feature_name].apply(lambda x: get_gender(x,0))
train[feature_name +'_genders_1'] = train[feature_name].apply(lambda x: get_gender(x,1))
train[feature_name +'_genders_2'] = train[feature_name].apply(lambda x: get_gender(x,2))
feature_list = ['cast','crew']
for feature_name in feature_list:
if feature_name == 'cast':
list_keys = ['name','character']
for i_key in list_keys:
feature = test[feature_name].apply(lambda x: all_data(x,i_key))
list_feature = test[feature_name].apply(lambda x: make_list(x,i_key))
list_of_feature = list(list_feature.values)
 top_feature_data = [c[0] for c in Counter([i for j in list_of_feature for i in j] ).most_common(10)]
feature_name_sv = singular_verb(feature_name)
 for name in top_feature_data:
test[feature_name_sv + '_'+ i_key + '_' + name] = feature.apply(lambda x:apply_encode(x,name))
test[feature_name_sv + '_'+ i_key + '_' + name] = test[feature_name_sv + '_'+ i_key + '_' + name].fillna(0 ).astype('int32')
 del feature
del list_feature
if feature_name == 'crew':
list_keys = ['name','job','department']
for i_key in list_keys:
feature = test[feature_name].apply(lambda x: all_data(x,i_key))
list_feature = test[feature_name].apply(lambda x: make_list(x,i_key))
list_of_feature = list(list_feature.values)
 top_feature_data = [c[0] for c in Counter([i for j in list_of_feature for i in j] ).most_common(10)]
feature_name_sv = singular_verb(feature_name)
 for name in top_feature_data:
test[feature_name_sv + '_'+ i_key + '_' + name] = feature.apply(lambda x:apply_encode(x,name))
test[feature_name_sv + '_'+ i_key + '_' + name] = test[feature_name_sv + '_'+ i_key + '_' + name].fillna(0 ).astype('int32')
 del feature
del list_feature
test[feature_name +'_genders_0'] = test[feature_name].apply(lambda x: get_gender(x,0))
test[feature_name +'_genders_1'] = test[feature_name].apply(lambda x: get_gender(x,1))
test[feature_name +'_genders_2'] = test[feature_name].apply(lambda x: get_gender(x,2))
install_AMT_PAYMENT_mean = install.groupby('SK_ID_PREV', as_index=False)['AMT_PAYMENT'].mean().rename(columns = {'AMT_PAYMENT': 'install_AMT_PAYMENT_mean'})
 install_stats_by_prev = install_stats_by_prev.merge(install_AMT_PAYMENT_mean, on = 'SK_ID_PREV', how = 'left')
 gc.enable()
del install_AMT_PAYMENT_mean
gc.collect()<merge>
train['release_date'] = train['release_date'].apply(lambda x: fix_date(x))
train['release_date'] = pd.to_datetime(train['release_date'])
 train = process_date(train)
 test['release_date'] = test['release_date'].apply(lambda x: fix_date(x))
test['release_date'] = pd.to_datetime(test['release_date'])
 test = process_date(test )
install_DAYS_DIFF_mean = install.groupby('SK_ID_PREV', as_index=False)['DAYS_DIFF'].mean().rename(columns = {'DAYS_DIFF': 'install_DAYS_DIFF_mean'})
 install_DAYS_DIFF_max = install.groupby('SK_ID_PREV', as_index=False)['DAYS_DIFF'].max().rename(columns = {'DAYS_DIFF': 'install_DAYS_DIFF_max'})
 install_DAYS_DIFF_min = install.groupby('SK_ID_PREV', as_index=False)['DAYS_DIFF'].min().rename(columns = {'DAYS_DIFF': 'install_DAYS_DIFF_min'})
 install_stats_by_prev = install_stats_by_prev.merge(install_DAYS_DIFF_mean, on = 'SK_ID_PREV', how = 'left')
 install_stats_by_prev = install_stats_by_prev.merge(install_DAYS_DIFF_max, on = 'SK_ID_PREV', how = 'left')
 install_stats_by_prev = install_stats_by_prev.merge(install_DAYS_DIFF_min, on = 'SK_ID_PREV', how = 'left')
 gc.enable()
del install_DAYS_DIFF_mean, install_DAYS_DIFF_max, install_DAYS_DIFF_min
gc.collect()<merge>
for col in ['original_language', 'collection_name']:
le = LabelEncoder()
le.fit(list(train[col].fillna('')) + list(test[col].fillna('')))
 train[col] = le.transform(train[col].fillna('' ).astype(str))
test[col] = le.transform(test[col].fillna('' ).astype(str))
install_AMT_DIFF_mean = install.groupby('SK_ID_PREV', as_index=False)['AMT_DIFF'].mean().rename(columns = {'AMT_DIFF': 'install_AMT_DIFF_mean'})
 install_AMT_DIFF_max = install.groupby('SK_ID_PREV', as_index=False)['AMT_DIFF'].max().rename(columns = {'AMT_DIFF': 'install_AMT_DIFF_max'})
 install_AMT_DIFF_min = install.groupby('SK_ID_PREV', as_index=False)['AMT_DIFF'].min().rename(columns = {'AMT_DIFF': 'install_AMT_DIFF_min'})
 install_stats_by_prev = install_stats_by_prev.merge(install_AMT_DIFF_mean, on = 'SK_ID_PREV', how = 'left')
 install_stats_by_prev = install_stats_by_prev.merge(install_AMT_DIFF_max, on = 'SK_ID_PREV', how = 'left')
 install_stats_by_prev = install_stats_by_prev.merge(install_AMT_DIFF_min, on = 'SK_ID_PREV', how = 'left')
 gc.enable()
del install_AMT_DIFF_mean, install_AMT_DIFF_max, install_AMT_DIFF_min
gc.collect()<merge>
train = train.drop(['belongs_to_collection','cast','crew','homepage', 'imdb_id', 'poster_path', 'status'], axis=1)
 test = test.drop(['belongs_to_collection','cast','crew','homepage', 'imdb_id', 'poster_path', 'status'], axis=1 )
install_NUM_INSTALMENT_VERSION_count_mean = install_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['install_NUM_INSTALMENT_VERSION_count'].mean().rename(columns = {'install_NUM_INSTALMENT_VERSION_count': 'install_NUM_INSTALMENT_VERSION_count_mean'})
 dataset = dataset.merge(install_NUM_INSTALMENT_VERSION_count_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del install_NUM_INSTALMENT_VERSION_count_mean
gc.collect()<merge>
for col in train.columns:
if train[col].nunique() == 1:
print(col)
 train = train.drop([col], axis=1)
 test = test.drop([col], axis=1 )
install_NUM_INSTALMENT_VERSION_max_max = install_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['install_NUM_INSTALMENT_VERSION_max'].max().rename(columns = {'install_NUM_INSTALMENT_VERSION_max': 'install_NUM_INSTALMENT_VERSION_max_max'})
 dataset = dataset.merge(install_NUM_INSTALMENT_VERSION_max_max, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del install_NUM_INSTALMENT_VERSION_max_max
gc.collect()<merge>
train=train.drop(['original_title', 'overview','release_date','tagline','title'], axis=1)
 test = test.drop(['original_title', 'overview','release_date','tagline','title'], axis=1 )
install_DAYS_INSTALMENT_mean_mean = install_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['install_DAYS_INSTALMENT_mean'].mean().rename(columns = {'install_DAYS_INSTALMENT_mean': 'install_DAYS_INSTALMENT_mean_mean'})
 dataset = dataset.merge(install_DAYS_INSTALMENT_mean_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del install_DAYS_INSTALMENT_mean_mean
gc.collect()<merge>
train.isnull().any().sum()
install_DAYS_ENTRY_PAYMENT_mean_mean = install_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['install_DAYS_ENTRY_PAYMENT_mean'].mean().rename(columns = {'install_DAYS_ENTRY_PAYMENT_mean': 'install_DAYS_ENTRY_PAYMENT_mean_mean'})
 dataset = dataset.merge(install_DAYS_ENTRY_PAYMENT_mean_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del install_DAYS_ENTRY_PAYMENT_mean_mean
gc.collect()<merge>
x_train = train.drop(['id', 'revenue'], axis=1 ).values
y_train = np.log1p(train['revenue'] ).values
test_id = test['id']
test = test.drop(['id'], axis=1 ).values
install_AMT_INSTALMENT_mean_mean = install_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['install_AMT_INSTALMENT_mean'].mean().rename(columns = {'install_AMT_INSTALMENT_mean': 'install_AMT_INSTALMENT_mean_mean'})
 dataset = dataset.merge(install_AMT_INSTALMENT_mean_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del install_AMT_INSTALMENT_mean_mean
gc.collect()<merge>
xg_reg = xgb.XGBRegressor(colsample_bytree = 0.3, learning_rate = 0.001,
max_depth = 3, alpha = 10, n_estimators = 15000 )
install_AMT_PAYMENT_mean_mean = install_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['install_AMT_PAYMENT_mean'].mean().rename(columns = {'install_AMT_PAYMENT_mean': 'install_AMT_PAYMENT_mean_mean'})
 dataset = dataset.merge(install_AMT_PAYMENT_mean_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del install_AMT_PAYMENT_mean_mean
gc.collect()<merge>
xg_reg.fit(x_train,y_train )
install_DAYS_DIFF_mean_mean = install_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['install_DAYS_DIFF_mean'].mean().rename(columns = {'install_DAYS_DIFF_mean': 'install_DAYS_DIFF_mean_mean'})
 dataset = dataset.merge(install_DAYS_DIFF_mean_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del install_DAYS_DIFF_mean_mean
gc.collect()<merge>
test_pred = xg_reg.predict(test)
 test_pred = np.expm1(test_pred )
<merge><EOS> 
dict = {'id': test_id, 'revenue': test_pred}
df = pd.DataFrame(dict)
 df.to_csv('my_submission_file.csv',index=False )