Kaggle competition: TMDB Box Office Prediction
<SOS> metric: RMSLE Kaggle data source: tmdb-box-office-prediction<merge>
py.init_notebook_mode(connected=True)
 plt.style.use('ggplot')
 print(os.listdir(".. /input"))
credit_AMT_INST_MIN_REGULARITY_mean = credit.groupby('SK_ID_PREV', as_index=False)['AMT_INST_MIN_REGULARITY'].mean().rename(columns = {'AMT_INST_MIN_REGULARITY': 'credit_AMT_INST_MIN_REGULARITY_mean'})
 credit_stats_by_prev = credit_stats_by_prev.merge(credit_AMT_INST_MIN_REGULARITY_mean, on = 'SK_ID_PREV', how = 'left')
 gc.enable()
del credit_AMT_INST_MIN_REGULARITY_mean
gc.collect()<merge>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 train_df.head()
credit_AMT_PAYMENT_CURRENT_mean = credit.groupby('SK_ID_PREV', as_index=False)['AMT_PAYMENT_CURRENT'].mean().rename(columns = {'AMT_PAYMENT_CURRENT': 'credit_AMT_PAYMENT_CURRENT_mean'})
 credit_stats_by_prev = credit_stats_by_prev.merge(credit_AMT_PAYMENT_CURRENT_mean, on = 'SK_ID_PREV', how = 'left')
 gc.enable()
del credit_AMT_PAYMENT_CURRENT_mean
gc.collect()<merge>
def fix_date(x):

 year = x.split('/')[2]
if int(year)<= 19:
return x[:-2] + '20' + year
else:
return x[:-2] + '19' + year
train_df['release_date'] = pd.to_datetime(train_df['release_date'].apply(lambda x: fix_date(x)))
 test_df.loc[test_df['release_date'].isnull() == True, 'release_date'] = '01/01/98'
test_df['release_date'] = pd.to_datetime(test_df['release_date'].apply(lambda x: fix_date(x)))
 train_df['release_date'].head()
credit_AMT_PAYMENT_TOTAL_CURRENT_mean = credit.groupby('SK_ID_PREV', as_index=False)['AMT_PAYMENT_TOTAL_CURRENT'].mean().rename(columns = {'AMT_PAYMENT_TOTAL_CURRENT': 'credit_AMT_PAYMENT_TOTAL_CURRENT_mean'})
 credit_stats_by_prev = credit_stats_by_prev.merge(credit_AMT_PAYMENT_TOTAL_CURRENT_mean, on = 'SK_ID_PREV', how = 'left')
 gc.enable()
del credit_AMT_PAYMENT_TOTAL_CURRENT_mean
gc.collect()<merge>
def process_date(df):
date_parts = ["year", "weekday", "month", 'weekofyear', 'day', 'quarter']
for part in date_parts:
part_col = 'release_date' + "_" + part
df[part_col] = getattr(df['release_date'].dt, part ).astype(int)
 return df
train_df = process_date(train_df)
 test_df = process_date(test_df )
credit_AMT_RECEIVABLE_PRINCIPAL_mean = credit.groupby('SK_ID_PREV', as_index=False)['AMT_RECEIVABLE_PRINCIPAL'].mean().rename(columns = {'AMT_RECEIVABLE_PRINCIPAL': 'credit_AMT_RECEIVABLE_PRINCIPAL_mean'})
 credit_stats_by_prev = credit_stats_by_prev.merge(credit_AMT_RECEIVABLE_PRINCIPAL_mean, on = 'SK_ID_PREV', how = 'left')
 gc.enable()
del credit_AMT_RECEIVABLE_PRINCIPAL_mean
gc.collect()<merge>
dict_columns = ['belongs_to_collection', 'genres', 'production_companies', 'production_countries',
'spoken_languages', 'Keywords', 'cast', 'crew']
def convert_dict(df):
for column in dict_columns:
df[column] = df[column].apply(lambda x: ast.literal_eval(x)if not pd.isnull(x)else np.NaN)
 return df
credit_AMT_RECIVABLE_mean = credit.groupby('SK_ID_PREV', as_index=False)['AMT_RECIVABLE'].mean().rename(columns = {'AMT_RECIVABLE': 'credit_AMT_RECIVABLE_mean'})
 credit_stats_by_prev = credit_stats_by_prev.merge(credit_AMT_RECIVABLE_mean, on = 'SK_ID_PREV', how = 'left')
 gc.enable()
del credit_AMT_RECIVABLE_mean
gc.collect()<merge>
train_df = convert_dict(train_df)
 test_df = convert_dict(test_df )
credit_AMT_TOTAL_RECEIVABLE_mean = credit.groupby('SK_ID_PREV', as_index=False)['AMT_TOTAL_RECEIVABLE'].mean().rename(columns = {'AMT_TOTAL_RECEIVABLE': 'credit_AMT_TOTAL_RECEIVABLE_mean'})
 credit_stats_by_prev = credit_stats_by_prev.merge(credit_AMT_TOTAL_RECEIVABLE_mean, on = 'SK_ID_PREV', how = 'left')
 gc.enable()
del credit_AMT_TOTAL_RECEIVABLE_mean
gc.collect()<merge>
train_df['belongs_to_collection'].isnull().sum()
credit_CNT_DRAWINGS_ATM_CURRENT_mean = credit.groupby('SK_ID_PREV', as_index=False)['CNT_DRAWINGS_ATM_CURRENT'].mean().rename(columns = {'CNT_DRAWINGS_ATM_CURRENT': 'credit_CNT_DRAWINGS_ATM_CURRENT_mean'})
 credit_stats_by_prev = credit_stats_by_prev.merge(credit_CNT_DRAWINGS_ATM_CURRENT_mean, on = 'SK_ID_PREV', how = 'left')
 gc.enable()
del credit_CNT_DRAWINGS_ATM_CURRENT_mean
gc.collect()<merge>
train_df['has_collection'] = train_df['belongs_to_collection'].apply(lambda x: 1 if not pd.isnull(x)else 0)
 test_df['has_collection'] = test_df['belongs_to_collection'].apply(lambda x: 1 if not pd.isnull(x)else 0)
 train_df['has_collection'][:5]
credit_CNT_DRAWINGS_CURRENT_mean = credit.groupby('SK_ID_PREV', as_index=False)['CNT_DRAWINGS_CURRENT'].mean().rename(columns = {'CNT_DRAWINGS_CURRENT': 'credit_CNT_DRAWINGS_CURRENT_mean'})
 credit_stats_by_prev = credit_stats_by_prev.merge(credit_CNT_DRAWINGS_CURRENT_mean, on = 'SK_ID_PREV', how = 'left')
 gc.enable()
del credit_CNT_DRAWINGS_CURRENT_mean
gc.collect()<merge>
train_df['collection_name'] = train_df['belongs_to_collection'].apply(lambda x: x[0]['name'] if not pd.isnull(x)else 0)
 test_df['collection_name'] = test_df['belongs_to_collection'].apply(lambda x: x[0]['name'] if not pd.isnull(x)else 0)
 train_df['collection_name'][:5]
credit_CNT_DRAWINGS_OTHER_CURRENT_mean = credit.groupby('SK_ID_PREV', as_index=False)['CNT_DRAWINGS_OTHER_CURRENT'].mean().rename(columns = {'CNT_DRAWINGS_OTHER_CURRENT': 'credit_CNT_DRAWINGS_OTHER_CURRENT_mean'})
 credit_stats_by_prev = credit_stats_by_prev.merge(credit_CNT_DRAWINGS_OTHER_CURRENT_mean, on = 'SK_ID_PREV', how = 'left')
 gc.enable()
del credit_CNT_DRAWINGS_OTHER_CURRENT_mean
gc.collect()<merge>
train_df = train_df.drop("belongs_to_collection", axis =1)
 test_df = test_df.drop("belongs_to_collection", axis =1 )
credit_CNT_DRAWINGS_POS_CURRENT_mean = credit.groupby('SK_ID_PREV', as_index=False)['CNT_DRAWINGS_POS_CURRENT'].mean().rename(columns = {'CNT_DRAWINGS_POS_CURRENT': 'credit_CNT_DRAWINGS_POS_CURRENT_mean'})
 credit_stats_by_prev = credit_stats_by_prev.merge(credit_CNT_DRAWINGS_POS_CURRENT_mean, on = 'SK_ID_PREV', how = 'left')
 gc.enable()
del credit_CNT_DRAWINGS_POS_CURRENT_mean
gc.collect()<merge>
def dictToInd(var, num):
train_df[var] = train_df[var].fillna(0)
 test_df[var] = test_df[var].fillna(0)
 train_df['num_' + var] = train_df[var].apply(lambda x: len(x)if x!= 0 else 0)
 test_df['num_' + var] = test_df[var].apply(lambda x: len(x)if x!= 0 else 0)
 train_df['all_'+var] = train_df[var].apply(lambda x: [i['name'] for i in x] if x !=0 else 0)
 test_df['all_'+var] = test_df[var].apply(lambda x: [i['name'] for i in x] if x !=0 else 0)
 d = dict()
for i, e in enumerate(train_df['all_' + var]):
if e != 0:
for k in e:
if k in d:
d[k] += 1
else:
d[k] = 1
top_d = dict(sorted(d.items() , key = operator.itemgetter(1), reverse = True)[:num])
 for i in top_d:
train_df[var +'_' + i] = train_df['all_' + var].apply(lambda x: 1 if x!=0 and i in x else 0)
 test_df[var +'_' + i] = test_df['all_' + var].apply(lambda x: 1 if x!=0 and i in x else 0 )
credit_CNT_INSTALMENT_MATURE_CUM_mean = credit.groupby('SK_ID_PREV', as_index=False)['CNT_INSTALMENT_MATURE_CUM'].mean().rename(columns = {'CNT_INSTALMENT_MATURE_CUM': 'credit_CNT_INSTALMENT_MATURE_CUM_mean'})
 credit_stats_by_prev = credit_stats_by_prev.merge(credit_CNT_INSTALMENT_MATURE_CUM_mean, on = 'SK_ID_PREV', how = 'left')
 gc.enable()
del credit_CNT_INSTALMENT_MATURE_CUM_mean
gc.collect()<merge>
train_df['num_genres'].value_counts()
credit_SK_DPD_mean = credit.groupby('SK_ID_PREV', as_index=False)['SK_DPD'].mean().rename(columns = {'SK_DPD': 'credit_SK_DPD_mean'})
 credit_stats_by_prev = credit_stats_by_prev.merge(credit_SK_DPD_mean, on = 'SK_ID_PREV', how = 'left')
 gc.enable()
del credit_SK_DPD_mean
gc.collect()<merge>
genres = []
for i, e in enumerate(train_df['all_genres']):
if e != 0:
for genre in e:
if genre not in genres:
genres.append(genre)
 print(genres )
credit_SK_DPD_DEF_mean = credit.groupby('SK_ID_PREV', as_index=False)['SK_DPD_DEF'].mean().rename(columns = {'SK_DPD_DEF': 'credit_SK_DPD_DEF_mean'})
 credit_stats_by_prev = credit_stats_by_prev.merge(credit_SK_DPD_DEF_mean, on = 'SK_ID_PREV', how = 'left')
 gc.enable()
del credit_SK_DPD_DEF_mean
gc.collect()<categorify>
train_df['production_companies'].fillna(0 ).apply(lambda x: len(x)if x != 0 else 0 ).value_counts()
credit_cats = pd.get_dummies(credit.select_dtypes('object'))
credit_cats['SK_ID_PREV'] = credit['SK_ID_PREV']
credit_cats.head()<groupby>
train_df['production_countries'].fillna(0 ).apply(lambda x: len(x)if x != 0 else 0 ).value_counts()
credit_cats_grouped = credit_cats.groupby('SK_ID_PREV' ).agg('sum')
 credit_cats_grouped.head()<merge>
train_df['spoken_languages'].fillna(0 ).apply(lambda x: len(x)if x != 0 else 0 ).value_counts()
credit_stats_by_prev = credit_stats_by_prev.merge(credit_cats_grouped, on = 'SK_ID_PREV', how = 'left')
 gc.enable()
del credit_cats_grouped, credit_cats
gc.collect()<merge>
train_df['Keywords'].fillna(0 ).apply(lambda x: len(x)if x !=0 else 0 ).value_counts()
credit_MONTHS_BALANCE_count_mean = credit_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['credit_MONTHS_BALANCE_count'].mean().rename(columns = {'credit_MONTHS_BALANCE_count': 'credit_MONTHS_BALANCE_count_mean'})
 dataset = dataset.merge(credit_MONTHS_BALANCE_count_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del credit_MONTHS_BALANCE_count_mean
gc.collect()<merge>
list_of_keywords = list(train_df['Keywords'].fillna(0 ).apply(lambda x: [i['name'] for i in x] if x != 0 else [] ).values)
 list_of_keywords[:19]
credit_MONTHS_BALANCE_mean_mean = credit_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['credit_MONTHS_BALANCE_mean'].mean().rename(columns = {'credit_MONTHS_BALANCE_mean': 'credit_MONTHS_BALANCE_mean_mean'})
 dataset = dataset.merge(credit_MONTHS_BALANCE_mean_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del credit_MONTHS_BALANCE_mean_mean
gc.collect()<merge>
train_df['cast'].fillna(0 ).apply(lambda x: len(x)if x != 0 else 0 ).value_counts()
credit_AMT_BALANCE_mean_mean = credit_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['credit_AMT_BALANCE_mean'].mean().rename(columns = {'credit_AMT_BALANCE_mean': 'credit_AMT_BALANCE_mean_mean'})
 dataset = dataset.merge(credit_AMT_BALANCE_mean_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del credit_AMT_BALANCE_mean_mean
gc.collect()<merge>
for i in train_df.columns:
num_nas = train_df[i].isnull().sum()
if num_nas > 0:
print(i, " :", num_nas )
credit_AMT_CREDIT_LIMIT_ACTUAL_mean_mean = credit_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['credit_AMT_CREDIT_LIMIT_ACTUAL_mean'].mean().rename(columns = {'credit_AMT_CREDIT_LIMIT_ACTUAL_mean': 'credit_AMT_CREDIT_LIMIT_ACTUAL_mean_mean'})
 dataset = dataset.merge(credit_AMT_CREDIT_LIMIT_ACTUAL_mean_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del credit_AMT_CREDIT_LIMIT_ACTUAL_mean_mean
gc.collect()<merge>
for i in test_df.columns:
num_nas = test_df[i].isnull().sum()
if num_nas > 0:
print(i, " :", num_nas )
credit_AMT_DRAWINGS_ATM_CURRENT_mean_mean = credit_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['credit_AMT_DRAWINGS_ATM_CURRENT_mean'].mean().rename(columns = {'credit_AMT_DRAWINGS_ATM_CURRENT_mean': 'credit_AMT_DRAWINGS_ATM_CURRENT_mean_mean'})
 dataset = dataset.merge(credit_AMT_DRAWINGS_ATM_CURRENT_mean_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del credit_AMT_DRAWINGS_ATM_CURRENT_mean_mean
gc.collect()<merge>
test_df['status'].value_counts()
credit_AMT_DRAWINGS_CURRENT_mean_mean = credit_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['credit_AMT_DRAWINGS_CURRENT_mean'].mean().rename(columns = {'credit_AMT_DRAWINGS_CURRENT_mean': 'credit_AMT_DRAWINGS_CURRENT_mean_mean'})
 dataset = dataset.merge(credit_AMT_DRAWINGS_CURRENT_mean_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del credit_AMT_DRAWINGS_CURRENT_mean_mean
gc.collect()<merge>
train_df['runtime'] = train_df['runtime'].fillna(train_df['runtime'].median())
 test_df['runtime'] = test_df['runtime'].fillna(test_df['runtime'].median())
 train_df['tagline'] = train_df['tagline'].fillna('')
 test_df['tagline'] = test_df['tagline'].fillna('')
 train_df['homepage'] = train_df['homepage'].fillna(0)
 test_df['homepage'] = test_df['homepage'].fillna(0)
 train_df['has_tagline'] = train_df['tagline'].apply(lambda x: 1 if x != '' else 0)
 test_df['has_tagline'] = test_df['tagline'].apply(lambda x: 1 if x != '' else 0)
 train_df['has_homepage'] = train_df['homepage'].apply(lambda x: 1 if x != 0 else 0)
 test_df['has_homepage'] = test_df['homepage'].apply(lambda x: 1 if x != 0 else 0)
 train_df['budget'] = train_df['budget'].apply(lambda x: x if x > 0 else np.NaN)
 test_df['budget'] = test_df['budget'].apply(lambda x: x if x > 0 else np.NaN)
 train_df['budget'] = train_df['budget'].fillna(train_df['budget'].median())
 test_df['budget'] = test_df['budget'].fillna(test_df['budget'].median())
 test_df['status'] = test_df['status'].fillna(test_df['status'].mode() [0])
credit_AMT_DRAWINGS_OTHER_CURRENT_mean_mean = credit_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['credit_AMT_DRAWINGS_OTHER_CURRENT_mean'].mean().rename(columns = {'credit_AMT_DRAWINGS_OTHER_CURRENT_mean': 'credit_AMT_DRAWINGS_OTHER_CURRENT_mean_mean'})
 dataset = dataset.merge(credit_AMT_DRAWINGS_OTHER_CURRENT_mean_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del credit_AMT_DRAWINGS_OTHER_CURRENT_mean_mean
gc.collect()<merge>
le = LabelEncoder()
train_df['status'] = le.fit_transform(train_df['status'])
 test_df['status'] = le.fit_transform(test_df['status'])
 train_df['status'].value_counts()
credit_AMT_DRAWINGS_POS_CURRENT_mean_mean = credit_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['credit_AMT_DRAWINGS_POS_CURRENT_mean'].mean().rename(columns = {'credit_AMT_DRAWINGS_POS_CURRENT_mean': 'credit_AMT_DRAWINGS_POS_CURRENT_mean_mean'})
 dataset = dataset.merge(credit_AMT_DRAWINGS_POS_CURRENT_mean_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del credit_AMT_DRAWINGS_POS_CURRENT_mean_mean
gc.collect()<merge>
train_df['log_revenue'] = np.log1p(train_df['revenue'])
 train_df['log_budget'] = np.log1p(train_df['budget'])
 test_df['log_budget'] = np.log1p(test_df['budget'] )
credit_AMT_INST_MIN_REGULARITY_mean_mean = credit_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['credit_AMT_INST_MIN_REGULARITY_mean'].mean().rename(columns = {'credit_AMT_INST_MIN_REGULARITY_mean': 'credit_AMT_INST_MIN_REGULARITY_mean_mean'})
 dataset = dataset.merge(credit_AMT_INST_MIN_REGULARITY_mean_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del credit_AMT_INST_MIN_REGULARITY_mean_mean
gc.collect()<merge>
vectorizer = TfidfVectorizer(
 sublinear_tf=True,
analyzer='word',
token_pattern=r'\w{1,}',
ngram_range=(1, 2),
min_df=5)
 overview_text = vectorizer.fit_transform(train_df['overview'].fillna(''))
linreg = LinearRegression()
linreg.fit(overview_text, train_df['log_revenue'])
 eli5.show_weights(linreg, vec=vectorizer, top=20, feature_filter=lambda x: x != '<BIAS>' )
credit_AMT_PAYMENT_CURRENT_mean_mean = credit_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['credit_AMT_PAYMENT_CURRENT_mean'].mean().rename(columns = {'credit_AMT_PAYMENT_CURRENT_mean': 'credit_AMT_PAYMENT_CURRENT_mean_mean'})
 dataset = dataset.merge(credit_AMT_PAYMENT_CURRENT_mean_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del credit_AMT_PAYMENT_CURRENT_mean_mean
gc.collect()<merge>
train_texts = train_df[['title', 'tagline', 'overview', 'original_title']]
test_texts = test_df[['title', 'tagline', 'overview', 'original_title']]
train_df[['title', 'tagline', 'overview', 'original_title']].head()
credit_AMT_PAYMENT_TOTAL_CURRENT_mean_mean = credit_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['credit_AMT_PAYMENT_TOTAL_CURRENT_mean'].mean().rename(columns = {'credit_AMT_PAYMENT_TOTAL_CURRENT_mean': 'credit_AMT_PAYMENT_TOTAL_CURRENT_mean_mean'})
 dataset = dataset.merge(credit_AMT_PAYMENT_TOTAL_CURRENT_mean_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del credit_AMT_PAYMENT_TOTAL_CURRENT_mean_mean
gc.collect()<merge>
for col in ['title','overview', 'tagline','original_title']:
train_df['len_' + col] = train_df[col].fillna('' ).apply(lambda x: len(str(x)))
 train_df['words_' + col] = train_df[col].fillna('' ).apply(lambda x: len(str(x.split(' '))))
train_df = train_df.drop(col, axis=1)
 test_df['len_' + col] = test_df[col].fillna('' ).apply(lambda x: len(str(x)))
 test_df['words_' + col] = test_df[col].fillna('' ).apply(lambda x: len(str(x.split(' '))))
test_df = test_df.drop(col, axis=1 )
credit_AMT_RECEIVABLE_PRINCIPAL_mean_mean = credit_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['credit_AMT_RECEIVABLE_PRINCIPAL_mean'].mean().rename(columns = {'credit_AMT_RECEIVABLE_PRINCIPAL_mean': 'credit_AMT_RECEIVABLE_PRINCIPAL_mean_mean'})
 dataset = dataset.merge(credit_AMT_RECEIVABLE_PRINCIPAL_mean_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del credit_AMT_RECEIVABLE_PRINCIPAL_mean_mean
gc.collect()<merge>
train_df = pd.concat([train_df, pd.get_dummies(train_df['release_date_weekday'], prefix='release_weekday')], axis=1)
 train_df = pd.concat([train_df, pd.get_dummies(train_df['release_date_month'], prefix='release_month')], axis=1)
 train_df = pd.concat([train_df, pd.get_dummies(train_df['release_date_quarter'], prefix='release_quarter')], axis=1)
 test_df = pd.concat([test_df, pd.get_dummies(test_df['release_date_weekday'], prefix='release_weekday')], axis=1)
 test_df = pd.concat([test_df, pd.get_dummies(test_df['release_date_month'], prefix='release_month')], axis=1)
 test_df = pd.concat([test_df, pd.get_dummies(test_df['release_date_quarter'], prefix='release_quarter')], axis=1 )
credit_AMT_RECIVABLE_mean_mean = credit_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['credit_AMT_RECIVABLE_mean'].mean().rename(columns = {'credit_AMT_RECIVABLE_mean': 'credit_AMT_RECIVABLE_mean_mean'})
 dataset = dataset.merge(credit_AMT_RECIVABLE_mean_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del credit_AMT_RECIVABLE_mean_mean
gc.collect()<merge>
def new_features(df):
df['budget_to_popularity'] = df['budget'] / df['popularity']
df['budget_to_runtime'] = df['budget'] / df['runtime']
df['_budget_year_ratio'] = df['budget'] /(df['release_date_year'] * df['release_date_year'])
 df['_releaseYear_popularity_ratio'] = df['release_date_year'] / df['popularity']
df['_releaseYear_popularity_ratio2'] = df['popularity'] / df['release_date_year']
df['runtime_to_mean_year'] = df['runtime'] / df.groupby("release_date_year")["runtime"].transform('mean')
 df['popularity_to_mean_year'] = df['popularity'] / df.groupby("release_date_year")["popularity"].transform('mean')
 df['budget_to_mean_year'] = df['budget'] / df.groupby("release_date_year")["budget"].transform('mean')
 return df
credit_AMT_TOTAL_RECEIVABLE_mean_mean = credit_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['credit_AMT_TOTAL_RECEIVABLE_mean'].mean().rename(columns = {'credit_AMT_TOTAL_RECEIVABLE_mean': 'credit_AMT_TOTAL_RECEIVABLE_mean_mean'})
 dataset = dataset.merge(credit_AMT_TOTAL_RECEIVABLE_mean_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del credit_AMT_TOTAL_RECEIVABLE_mean_mean
gc.collect()<merge>
train_df =new_features(train_df)
 test_df =new_features(test_df)
 train_df.head()
credit_CNT_DRAWINGS_ATM_CURRENT_mean_mean = credit_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['credit_CNT_DRAWINGS_ATM_CURRENT_mean'].mean().rename(columns = {'credit_CNT_DRAWINGS_ATM_CURRENT_mean': 'credit_CNT_DRAWINGS_ATM_CURRENT_mean_mean'})
 dataset = dataset.merge(credit_CNT_DRAWINGS_ATM_CURRENT_mean_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del credit_CNT_DRAWINGS_ATM_CURRENT_mean_mean
gc.collect()<merge>
y = train_df['log_revenue']
X = train_df.drop(['id', 'genres', 'homepage', 'imdb_id','original_language', 'poster_path', 'production_companies', 'production_countries', 'spoken_languages',
'Keywords', 'cast', 'crew', 'revenue', 'collection_name','all_genres', 'all_production_companies', 'all_production_countries', 'all_spoken_languages',
'all_Keywords', 'all_cast', 'all_crew', 'release_date', 'revenue', 'log_revenue', 'budget', 'release_date_quarter',
'release_date_day', 'release_date_weekofyear', 'release_date_month', 'release_date_weekday',
'budget_to_runtime'
], axis = 1)
 X_test= test_df.drop(['id', 'genres', 'homepage', 'imdb_id','original_language', 'poster_path', 'production_companies', 'production_countries', 'spoken_languages',
'Keywords', 'cast', 'crew', 'collection_name','all_genres', 'all_production_companies', 'all_production_countries', 'all_spoken_languages',
'all_Keywords', 'all_cast', 'all_crew', 'release_date', 'budget', 'release_date_quarter', 'release_date_day',
'release_date_weekofyear', 'release_date_month', 'release_date_weekday',
'budget_to_runtime'
], axis = 1)
 X.head()
credit_CNT_DRAWINGS_CURRENT_mean_mean = credit_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['credit_CNT_DRAWINGS_CURRENT_mean'].mean().rename(columns = {'credit_CNT_DRAWINGS_CURRENT_mean': 'credit_CNT_DRAWINGS_CURRENT_mean_mean'})
 dataset = dataset.merge(credit_CNT_DRAWINGS_CURRENT_mean_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del credit_CNT_DRAWINGS_CURRENT_mean_mean
gc.collect()<merge>
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, random_state=1 )
credit_CNT_DRAWINGS_OTHER_CURRENT_mean_mean = credit_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['credit_CNT_DRAWINGS_OTHER_CURRENT_mean'].mean().rename(columns = {'credit_CNT_DRAWINGS_OTHER_CURRENT_mean': 'credit_CNT_DRAWINGS_OTHER_CURRENT_mean_mean'})
 dataset = dataset.merge(credit_CNT_DRAWINGS_OTHER_CURRENT_mean_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del credit_CNT_DRAWINGS_OTHER_CURRENT_mean_mean
gc.collect()<merge>
simple_forest = RandomForestRegressor(criterion = 'mse',
n_estimators =100,
random_state = 1 )
credit_CNT_DRAWINGS_POS_CURRENT_mean_mean = credit_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['credit_CNT_DRAWINGS_POS_CURRENT_mean'].mean().rename(columns = {'credit_CNT_DRAWINGS_POS_CURRENT_mean': 'credit_CNT_DRAWINGS_POS_CURRENT_mean_mean'})
 dataset = dataset.merge(credit_CNT_DRAWINGS_POS_CURRENT_mean_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del credit_CNT_DRAWINGS_POS_CURRENT_mean_mean
gc.collect()<merge>
simple_forest.fit(X_train, y_train)
 simple_forest_pred = simple_forest.predict(X_valid )
credit_CNT_INSTALMENT_MATURE_CUM_mean_mean = credit_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['credit_CNT_INSTALMENT_MATURE_CUM_mean'].mean().rename(columns = {'credit_CNT_INSTALMENT_MATURE_CUM_mean': 'credit_CNT_INSTALMENT_MATURE_CUM_mean_mean'})
 dataset = dataset.merge(credit_CNT_INSTALMENT_MATURE_CUM_mean_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del credit_CNT_INSTALMENT_MATURE_CUM_mean_mean
gc.collect()<merge>
mse = mean_squared_error(simple_forest_pred, y_valid)
 mse
credit_SK_DPD_mean_mean = credit_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['credit_SK_DPD_mean'].mean().rename(columns = {'credit_SK_DPD_mean': 'credit_SK_DPD_mean_mean'})
 dataset = dataset.merge(credit_SK_DPD_mean_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del credit_SK_DPD_mean_mean
gc.collect()<merge>
simple_forest_pred = simple_forest.predict(X_test)
 simple_forest_rev = np.expm1(simple_forest_pred)
 os.chdir('.. /working')
 submit = pd.DataFrame({'Id': test_df['id'], 'revenue': simple_forest_rev})
 submit.to_csv('submission_simple_forest.csv', index=False)
 submit.head()
credit_SK_DPD_DEF_mean_mean = credit_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['credit_SK_DPD_DEF_mean'].mean().rename(columns = {'credit_SK_DPD_DEF_mean': 'credit_SK_DPD_DEF_mean_mean'})
 dataset = dataset.merge(credit_SK_DPD_DEF_mean_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del credit_SK_DPD_DEF_mean_mean
gc.collect()<merge>
params = {'num_leaves': 30,
'min_data_in_leaf': 20,
'objective': 'regression',
'max_depth': 5,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.2,
"verbosity": -1}
light_gbm = lgb.LGBMRegressor(**params, n_estimators = 20000, nthread = 4, n_jobs = -1, random_state=1)
 light_gbm.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='rmse',
verbose=1000, early_stopping_rounds=200 )
credit_NAME_CONTRACT_STATUS_Active_mean = credit_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['NAME_CONTRACT_STATUS_Active'].mean().rename(columns = {'NAME_CONTRACT_STATUS_Active': 'credit_NAME_CONTRACT_STATUS_Active_mean'})
 dataset = dataset.merge(credit_NAME_CONTRACT_STATUS_Active_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del credit_NAME_CONTRACT_STATUS_Active_mean
gc.collect()<merge>
eli5.show_weights(light_gbm, feature_filter=lambda x: x != '<BIAS>' )
credit_NAME_CONTRACT_STATUS_Approved_mean = credit_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['NAME_CONTRACT_STATUS_Approved'].mean().rename(columns = {'NAME_CONTRACT_STATUS_Approved': 'credit_NAME_CONTRACT_STATUS_Approved_mean'})
 dataset = dataset.merge(credit_NAME_CONTRACT_STATUS_Approved_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del credit_NAME_CONTRACT_STATUS_Approved_mean
gc.collect()<merge>
light_gbm_pred = light_gbm.predict(X_test)
 light_gbm_rev = np.expm1(light_gbm_pred)
 os.chdir('.. /working')
 submit = pd.DataFrame({'Id': test_df['id'], 'revenue': light_gbm_rev})
 submit.to_csv('submission_light_gbm.csv', index=False)
 submit.head()
credit_NAME_CONTRACT_STATUS_Completed_mean = credit_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['NAME_CONTRACT_STATUS_Completed'].mean().rename(columns = {'NAME_CONTRACT_STATUS_Completed': 'credit_NAME_CONTRACT_STATUS_Completed_mean'})
 dataset = dataset.merge(credit_NAME_CONTRACT_STATUS_Completed_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del credit_NAME_CONTRACT_STATUS_Completed_mean
gc.collect()<merge>
n_estimators = [int(x)for x in np.linspace(start = 200, stop = 2000, num = 10)]
max_features = ['auto', 'sqrt']
max_depth = [int(x)for x in np.linspace(10, 110, num = 11)]
max_depth.append(None)
 min_samples_split = [2, 5, 8, 10, 15]
min_samples_leaf = [1, 3, 5, 7, 10]
bootstrap = [True, False]
random_grid = {'n_estimators': n_estimators,
'max_features': max_features,
'max_depth': max_depth,
'min_samples_split': min_samples_split,
'min_samples_leaf': min_samples_leaf,
'bootstrap': bootstrap}
credit_NAME_CONTRACT_STATUS_Demand_mean = credit_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['NAME_CONTRACT_STATUS_Demand'].mean().rename(columns = {'NAME_CONTRACT_STATUS_Demand': 'credit_NAME_CONTRACT_STATUS_Demand_mean'})
 dataset = dataset.merge(credit_NAME_CONTRACT_STATUS_Demand_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del credit_NAME_CONTRACT_STATUS_Demand_mean
gc.collect()<merge>
rf_best = RandomForestRegressor(n_estimators=1400,
max_features= 'auto',
max_depth= 80,
min_samples_split= 2,
min_samples_leaf= 3,
bootstrap= True,
random_state=1 )
credit_NAME_CONTRACT_STATUS_Refused_mean = credit_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['NAME_CONTRACT_STATUS_Refused'].mean().rename(columns = {'NAME_CONTRACT_STATUS_Refused': 'credit_NAME_CONTRACT_STATUS_Refused_mean'})
 dataset = dataset.merge(credit_NAME_CONTRACT_STATUS_Refused_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del credit_NAME_CONTRACT_STATUS_Refused_mean
gc.collect()<merge>
rf_best.fit(X_train, y_train)
 rf_best_pred = rf_best.predict(X_valid)
 mse_best= mean_squared_error(rf_best_pred, y_valid)
 mse_best
credit_NAME_CONTRACT_STATUS_Sent_mean = credit_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['NAME_CONTRACT_STATUS_Sent proposal'].mean().rename(columns = {'NAME_CONTRACT_STATUS_Sent proposal': 'credit_NAME_CONTRACT_STATUS_Sent_mean'})
 dataset = dataset.merge(credit_NAME_CONTRACT_STATUS_Sent_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del credit_NAME_CONTRACT_STATUS_Sent_mean
gc.collect()<merge>
rf_best_pred = rf_best.predict(X_test)
 rf_best_pred[:5]
credit_NAME_CONTRACT_STATUS_Signed_mean = credit_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['NAME_CONTRACT_STATUS_Signed'].mean().rename(columns = {'NAME_CONTRACT_STATUS_Signed': 'credit_NAME_CONTRACT_STATUS_Signed_mean'})
 dataset = dataset.merge(credit_NAME_CONTRACT_STATUS_Signed_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del credit_NAME_CONTRACT_STATUS_Signed_mean
gc.collect()<set_options>
rf_best_rev = np.expm1(rf_best_pred)
 os.chdir('.. /working')
 submit = pd.DataFrame({'Id': test_df['id'], 'revenue': rf_best_rev})
 submit.to_csv('submission_rf_best.csv', index=False)
 submit.head()