Kaggle competition: TMDB Box Office Prediction
<SOS> metric: RMSLE Kaggle data source: tmdb-box-office-prediction<load_from_csv>
pd.set_option('display.max_columns', None)
 %matplotlib inline
stop = stopwords.words('english')
 scaler = MinMaxScaler(feature_range=(0,1))
df_application = pd.read_csv('.. /input/application_train.csv')
 df_application_test = pd.read_csv('.. /input/application_test.csv')
 df_application.head()<categorify>
train_df = pd.read_csv('.. /input/tmdb-clean-dataset/train.csv')
 test_df = pd.read_csv('.. /input/tmdb-clean-dataset/test.csv')
 merged_df = train_df.append(test_df,ignore_index=True )
df_application['Source'] = 'Train'
df_application_test['Source'] = 'Test'
df = pd.concat(( df_application,df_application_test),axis = 0,sort = False)
 cat_cols = [col for col in df.columns if(df[col].dtype == object)&(col != 'Source')]
le = preprocessing.LabelEncoder()
for col in cat_cols:
df[col] = le.fit_transform(df[col].fillna("Missing"))
df.head()
df_train = df[df['Source'] == "Train"].drop('Source', axis =1)
 df_test = df[df['Source'] == "Test"].drop('Source', axis =1)
 del df<load_from_csv>

 def get_count_houses(data_column,col_name):
ls = []
data = [extract_meta_2(str(row),col_name)for row in data_column]
data_2 = [house for production_houses in data for house in production_houses]
house_count = Counter(data_2)
 return house_count
def extract_single_meta(json_data,col_name):
if(json_data == 'nan'):
return 'x'
json_data = json_data.replace("'",'"')
 json_data = json_data.replace('None','"None"')
 json_data = json_data.replace('s n"','s n ')
 json_data = json_data.replace('ed"','ed ')
 json_data = json_data.replace('ld"','ld ')
 json_data = json_data.replace('n"s','ns')
 json_data = json_data.replace('o"s','os')
 json_data = json_data.replace('"s','s')
 data = json.loads(json_data)
 meta = [d[col_name] for d in data]
return meta[0]
def extract_meta(json_data,col_name):
if(json_data == 'nan'):
return 'x'
json_data = json_data.replace("'",'"')
 str_json_data = str(json_data)
 str_json_data = str_json_data.replace("l"A","l")
 data = None
data= json.loads(str_json_data)
 meta = [d[col_name] for d in data]
return meta
def extract_meta_2(json_data,col_name):
if(json_data == 'nan'):
return 'x'
json_data = json_data.replace(u'\xa0',u' ')
 json_data = json_data.replace("{'",'{"')
 json_data = json_data.replace("'}",'"}')
 json_data = json_data.replace("':",'":')
 json_data = json_data.replace(": '",': "')
 json_data = json_data.replace("',",'",')
 json_data = json_data.replace(", '",', "')
 json_data = json_data.replace("'",'')
 str_json_data = str(json_data)
 data= None
data= json.loads(str_json_data)
 meta = [d[col_name] for d in data]
return meta
def extract_metadata(data_column, col_name, col_type = "blank"):
all_meta = set()
counter = Counter()
if(col_type != 'production_companies' and col_type != 'keyword'):
distinct_meta = [extract_meta(str(row),col_name)for row in data_column]
else:
counter = get_count_houses(data_column,col_name)
 counter = {k:counter[k] for k in counter if counter[k] > 5}
distinct_meta = [extract_meta_2(str(row),col_name)for row in data_column]
flattened_meta = [data for row in distinct_meta for data in row]
distinct_meta = []
for k in counter:
if k in flattened_meta:
distinct_meta.append(k)
 for data in distinct_meta:
if(col_type != 'production_companies' and col_type != 'keyword'):
all_meta.update(set(data))
else:
all_meta.update(set([data]))
all_meta_dict = dict()
i = 0
for item in all_meta:
all_meta_dict[item] = i
i += 1
all_meta_list = list(all_meta)
 return all_meta, all_meta_dict, all_meta_list
def add_meta_to_dataframe(all_meta_dict,all_meta_list, _df, data_column, col_name, col_type='blank'):
if(col_type != 'production_companies' and col_type != 'keyword'):
distinct_meta = [extract_meta(str(row),col_name)for row in data_column]
else:
distinct_meta = [extract_meta_2(str(row),col_name)for row in data_column]
one_hot_np = np.empty([len(_df),len(all_meta_dict)])
 for i in range(0, len(_df)) :
one_hot_meta = np.zeros(len(all_meta_dict))
for j in range(0,len(distinct_meta[i])) :
if(distinct_meta[i][j] in all_meta_dict):
one_hot_meta[all_meta_dict[distinct_meta[i][j]]] = 1
one_hot_np[i] = one_hot_meta
one_hot_df = pd.DataFrame(one_hot_np)
 one_hot_df.columns = all_meta_list
if(col_name == 'iso_639_1'):
one_hot_df = one_hot_df['en']
else:
one_hot_df['sum' + col_name] = one_hot_df.sum(axis=1,skipna=False)
 _df_new = pd.concat([_df.reset_index(drop=True),one_hot_df],axis=1)
 return _df_new
genres = merged_df['genres']
genres.fillna('nan',inplace=True)
 all_meta, all_meta_dict, all_meta_list = extract_metadata(genres,"name")
 genres = train_df['genres']
genres.fillna('nan',inplace=True)
 train_df_new = add_meta_to_dataframe(all_meta_dict, all_meta_list, train_df, genres,"name")
 genres_test = test_df['genres']
genres_test.fillna('nan',inplace=True)
 test_df_new = add_meta_to_dataframe(all_meta_dict,all_meta_list,test_df,genres_test,"name")
 spoken_languages = merged_df['spoken_languages']
spoken_languages.fillna('nan',inplace=True)
 all_meta, all_meta_dict, all_meta_list = extract_metadata(spoken_languages,"iso_639_1")
 spoken_languages = train_df['spoken_languages']
spoken_languages.fillna('nan',inplace=True)
 train_df_new = add_meta_to_dataframe(all_meta_dict, all_meta_list, train_df_new, spoken_languages,"iso_639_1")
 spoken_languages_test = test_df['spoken_languages']
spoken_languages_test.fillna('nan',inplace=True)
 test_df_new = add_meta_to_dataframe(all_meta_dict,all_meta_list,test_df_new,spoken_languages_test,"iso_639_1")
 production_countries = merged_df['production_countries']
production_countries.fillna('nan',inplace=True)
 all_meta, all_meta_dict, all_meta_list = extract_metadata(production_countries,"iso_3166_1")
 production_countries = train_df['production_countries']
production_countries.fillna('nan',inplace=True)
 train_df_new = add_meta_to_dataframe(all_meta_dict, all_meta_list, train_df_new, production_countries,"iso_3166_1")
 production_countries_test = test_df['production_countries']
production_countries_test.fillna('nan',inplace=True)
 test_df_new = add_meta_to_dataframe(all_meta_dict,all_meta_list,test_df_new,production_countries_test,"iso_3166_1")

belongs_to_collection = train_df['belongs_to_collection']
belongs_to_collection.fillna('nan',inplace=True)
 result = [extract_single_meta(str(row),'name')for row in belongs_to_collection]
result = pd.DataFrame(result)
 result.columns = ['collection']
train_df_new = pd.concat([train_df_new.reset_index(drop=True),result],axis=1)
 belongs_to_collection_test = test_df['belongs_to_collection']
belongs_to_collection_test.fillna('nan',inplace=True)
 result = [extract_single_meta(str(row),'name')for row in belongs_to_collection_test]
result = pd.DataFrame(result)
 result.columns = ['collection']
test_df_new = pd.concat([test_df_new.reset_index(drop=True),result],axis=1)
 production_companies = merged_df['production_companies']
production_companies.fillna('nan',inplace=True)
 all_meta, all_meta_dict, all_meta_list = extract_metadata(production_companies,"name","production_companies")
 production_companies = train_df['production_companies']
production_companies.fillna('nan',inplace=True)
 train_df_new = add_meta_to_dataframe(all_meta_dict, all_meta_list, train_df_new, production_companies,"name","production_companies")
 production_companies_test = test_df['production_companies']
production_companies_test.fillna('nan',inplace=True)
 test_df_new = add_meta_to_dataframe(all_meta_dict,all_meta_list,test_df_new,production_companies_test,"name","production_companies")
df_bureau = pd.read_csv(".. /input/bureau.csv")
 df_bureau_balance = pd.read_csv(".. /input/bureau_balance.csv")
 df_bureau_balance["MONTHS_BALANCE"]= np.abs(df_bureau_balance["MONTHS_BALANCE"])
 df_bureau_balance["Period"] = np.where(( df_bureau_balance["MONTHS_BALANCE"] < 7),"short",np.where(( df_bureau_balance["MONTHS_BALANCE"] < 13),"medium","long"))
df_bureau_balance["Period_status"] = df_bureau_balance["Period"].astype(str)+ "_" + df_bureau_balance["STATUS"]
df_bureau_balance.head(5)
<groupby>

 def extract_single_meta(json_data,col_name):
if(json_data == 'nan'):
return 'x'
json_data = json_data.replace("'",'"')
 json_data = json_data.replace('None','"None"')
 json_data = json_data.replace('s n"','s n ')
 json_data = json_data.replace('ed"','ed ')
 json_data = json_data.replace('ld"','ld ')
 json_data = json_data.replace('n"s','ns')
 json_data = json_data.replace('o"s','os')
 json_data = json_data.replace('"s','s')
 data = json.loads(json_data)
 meta = [d for d in data]
return meta[0]

 imdb_id = train_df['imdb_id']
cast = train_df['cast']
main_ls = []
x = 0
i = 0
for item in cast:
i+= 1
ls = []
if str(item)== '[]':
main_ls.append(['-1','nan','nan','-1','-1','nan','10000','nan'])
 x += 1
continue
item = str(item)
 item = item.replace("', '","'; '")
 item = item.replace(", '","; '")
 item= item.replace("{","")
 item= item.replace("},",";")
 item= item.replace("[","")
 item= item.replace("]","")
 item= item.replace("'","")
 item = item.replace('Jimmy;,','Jimmy')
 item = item.replace('Elektra Quartet;','Elektra Quartet')
 item_ls = item.split(';')
 i = 0
ls = []
for it in item_ls:
i += 1
it = it.replace('[','')
 it = it.replace(']','')
 it_ls = it.split(':')
 ls.append(str(it_ls[1] ).strip())
 if i % 8 == 0:
ls.append(imdb_id[x])
 main_ls.append(ls)
 ls = []
x += 1
cast_df = DataFrame.from_records(main_ls)
 cast_df.columns = ['cast_id','character','credit_id','gender','id','name','order','profile_path','imdb_id']

 imdb_id = test_df['imdb_id']
cast = test_df['cast']
main_ls = []
x = 0
i = 0
for item in cast:
i+= 1
ls = []
if str(item)== '[]':
main_ls.append(['-1','nan','nan','-1','-1','nan','10000','nan'])
 x += 1
continue
item = str(item)
 item = item.replace("', '","'; '")
 item = item.replace(", '","; '")
 item= item.replace("{","")
 item= item.replace("},",";")
 item= item.replace("[","")
 item= item.replace("]","")
 item= item.replace("'","")
 item = item.replace('Jimmy;,','Jimmy')
 item = item.replace('Elektra Quartet;','Elektra Quartet')
 item_ls = item.split(';')
 i = 0
ls = []
for it in item_ls:
i += 1
it = it.replace('[','')
 it = it.replace(']','')
 it_ls = it.split(':')
 ls.append(str(it_ls[1] ).strip())
 if i % 8 == 0:
ls.append(imdb_id[x])
 main_ls.append(ls)
 ls = []
x += 1
cast_df_test = DataFrame.from_records(main_ls)
 cast_df_test.columns = ['cast_id','character','credit_id','gender','id','name','order','profile_path','imdb_id']
df_bureau_balance = df_bureau_balance.groupby(["SK_ID_BUREAU","Period_status"])\ 
.agg({"MONTHS_BALANCE" : ["count","min","max","mean"]})\ 
.reset_index()
df_bureau_balance.columns = [''.join(col ).strip() for col in df_bureau_balance.columns.values]
df_bureau_balance.head()<merge>


imdb_id = train_df['imdb_id']
cast = train_df['crew']
main_ls = []
x = 0
i = 0
for item in cast:
i+= 1
ls = []
if str(item)== '[]':
main_ls.append(['nan','nan','nan','nan','nan','nan','nan'])
 x += 1
continue
item = str(item)
 item = item.replace("', '","'; '")
 item = item.replace(", '","; '")
 item= item.replace("{","")
 item= item.replace("},",";")
 item= item.replace("[","")
 item= item.replace("]","")
 item= item.replace("'","")
 item = item.replace('Jimmy;,','Jimmy')
 item = item.replace('Elektra Quartet;','Elektra Quartet')
 item_ls = item.split(';')
 i = 0
ls = []
for it in item_ls:
i += 1
it = it.replace('[','')
 it = it.replace(']','')
 it_ls = it.split(':')
 ls.append(str(it_ls[1] ).strip())
 if i % 7 == 0:
ls.append(imdb_id[x])
 main_ls.append(ls)
 ls = []
x += 1
crew_df = DataFrame.from_records(main_ls)
 crew_df.columns = ['credit_id','department','gender','id','job','name','profile_path','imdb_id']

 imdb_id = test_df['imdb_id']
cast = test_df['crew']
main_ls = []
x = 0
i = 0
for item in cast:
i+= 1
ls = []
if str(item)== '[]':
main_ls.append(['nan','nan','nan','nan','nan','nan','nan'])
 x += 1
continue
item = str(item)
 item = item.replace("', '","'; '")
 item = item.replace(", '","; '")
 item= item.replace("{","")
 item= item.replace("},",";")
 item= item.replace("[","")
 item= item.replace("]","")
 item= item.replace("'","")
 item = item.replace('Jimmy;,','Jimmy')
 item = item.replace('Elektra Quartet;','Elektra Quartet')
 item_ls = item.split(';')
 i = 0
ls = []
for it in item_ls:
i += 1
it = it.replace('[','')
 it = it.replace(']','')
 it_ls = it.split(':')
 ls.append(str(it_ls[1] ).strip())
 if i % 7 == 0:
ls.append(imdb_id[x])
 main_ls.append(ls)
 ls = []
x += 1
crew_df_test = DataFrame.from_records(main_ls)
 crew_df_test.columns = ['credit_id','department','gender','id','job','name','profile_path','imdb_id']
df_bureau_balance = df_bureau_balance.pivot_table(index = 'SK_ID_BUREAU',columns = 'Period_status',values = ['MONTHS_BALANCEcount','MONTHS_BALANCEmin','MONTHS_BALANCEmax','MONTHS_BALANCEmean'] ).reset_index()
df_bureau_balance.columns = [''.join(col ).strip() for col in df_bureau_balance.columns.values]
df_bureau = pd.merge(df_bureau,df_bureau_balance, how="left", on = "SK_ID_BUREAU")
 df_bureau=pd.get_dummies(df_bureau,prefix =['CREDIT_ACTIVE','CREDIT_CURRENCY','CREDIT_TYPE'] ,columns = ['CREDIT_ACTIVE','CREDIT_CURRENCY','CREDIT_TYPE'],dummy_na = True)
 df_bureau.head(5 )<groupby>


cast_counts = cast_df[['cast_id','imdb_id']].groupby('imdb_id' ).agg('count')
 cast_counts.columns = ['cast_counts']
cast_counts.reset_index(inplace=True)
 df_train= train_df_new.merge(cast_counts, how='left',left_on='imdb_id',right_on='imdb_id')
 cast_gender_counts = cast_df[['cast_id', 'gender','imdb_id']].groupby(['imdb_id','gender'],as_index=False ).agg('count')
 cast_gender_counts = cast_gender_counts.pivot(index='imdb_id',columns='gender',values='cast_id')
 cast_gender_counts.reset_index(inplace=True)
 cast_gender_counts.fillna(0,inplace=True)
 cast_gender_counts.columns=['imdb_id','gender_cast_count_0','gender_cast_count_1','gender_cast_count_2']
df_train = df_train.merge(cast_gender_counts,how='left',left_on='imdb_id',right_on='imdb_id')
 cast_gender_top_3 = cast_df[['order','gender','imdb_id']]
cast_gender_top_3['int_order'] = cast_gender_top_3.order.astype(int)
 cast_gender_top_3 = cast_gender_top_3.loc[cast_gender_top_3['int_order'] <= 2,]
cast_gender_top_3.drop('order',inplace=True,axis=1)
 cast_gender_top_3 = cast_gender_top_3.pivot_table(index='imdb_id',columns='int_order',values='gender',aggfunc='first')
 cast_gender_top_3.reset_index(inplace=True)
 cast_gender_top_3.columns=['imdb_id','cast_1_gender','cast_2_gender','cast_3_gender']
df_train = df_train.merge(cast_gender_top_3,how='left',left_on='imdb_id',right_on='imdb_id')
 top_actors = pd.read_csv('.. /input/top-1000-tmdb-actors/Top 1000 Actors and Actresses.csv',encoding='iso-8859-1')
 top_actors_name = top_actors[['Name']]
cast_df = cast_df.merge(top_actors_name,how='left',left_on='name',right_on='Name')
 print(cast_df.head(10))
cast_top_actors = cast_df[['imdb_id','Name']]
cast_top_actors = cast_top_actors.dropna()
cast_top_actors_2= cast_top_actors.groupby("imdb_id" ).count()
cast_top_actors_2.reset_index(inplace=True)
 cast_top_actors_2.columns=['imdb_id','count_top_actors']
df_train = df_train.merge(cast_top_actors_2,how='left',left_on='imdb_id',right_on='imdb_id')

cast_counts = cast_df_test[['cast_id','imdb_id']].groupby('imdb_id' ).agg('count')
 cast_counts.columns = ['cast_counts']
cast_counts.reset_index(inplace=True)
 df_test= test_df_new.merge(cast_counts, how='left',left_on='imdb_id',right_on='imdb_id')
 cast_gender_counts = cast_df_test[['cast_id', 'gender','imdb_id']].groupby(['imdb_id','gender'],as_index=False ).agg('count')
 cast_gender_counts = cast_gender_counts.pivot(index='imdb_id',columns='gender',values='cast_id')
 cast_gender_counts.reset_index(inplace=True)
 cast_gender_counts.fillna(0,inplace=True)
 cast_gender_counts.columns=['imdb_id','gender_cast_count_0','gender_cast_count_1','gender_cast_count_2']
df_test = df_test.merge(cast_gender_counts,how='left',left_on='imdb_id',right_on='imdb_id')
 cast_gender_top_3 = cast_df_test[['order','gender','imdb_id']]
cast_gender_top_3['int_order'] = cast_gender_top_3.order.astype(int)
 cast_gender_top_3 = cast_gender_top_3.loc[cast_gender_top_3['int_order'] <= 2,]
cast_gender_top_3.drop('order',inplace=True,axis=1)
 cast_gender_top_3 = cast_gender_top_3.pivot_table(index='imdb_id',columns='int_order',values='gender',aggfunc='first')
 cast_gender_top_3.reset_index(inplace=True)
 cast_gender_top_3.columns=['imdb_id','cast_1_gender','cast_2_gender','cast_3_gender']
df_test = df_test.merge(cast_gender_top_3,how='left',left_on='imdb_id',right_on='imdb_id')
 top_actors = pd.read_csv('.. /input/top-1000-tmdb-actors/Top 1000 Actors and Actresses.csv',encoding='iso-8859-1')
 top_actors_name = top_actors[['Name']]
cast_df_test = cast_df_test.merge(top_actors_name,how='left',left_on='name',right_on='Name')
 cast_top_actors = cast_df_test[['imdb_id','Name']]
cast_top_actors = cast_top_actors.dropna()
cast_top_actors_2= cast_top_actors.groupby("imdb_id" ).count()
cast_top_actors_2.reset_index(inplace=True)
 cast_top_actors_2.columns=['imdb_id','count_top_actors']
df_test = df_test.merge(cast_top_actors_2,how='left',left_on='imdb_id',right_on='imdb_id' )
df_bureau_features = df_bureau.groupby(['SK_ID_CURR'])\ 
.agg({'SK_ID_BUREAU' : ['nunique'],
'DAYS_CREDIT' : ['min','max','mean','std'],
'CREDIT_DAY_OVERDUE' :['min','max','mean','std'],
'DAYS_CREDIT_ENDDATE':['min','max','mean','std'],
'DAYS_ENDDATE_FACT': ['min','max','mean','std'],
'AMT_CREDIT_MAX_OVERDUE' : ['mean','min','max'],
'CNT_CREDIT_PROLONG' : ['mean','min','max'],
'AMT_CREDIT_SUM' : ['min','max','mean','std'],
'AMT_CREDIT_SUM_DEBT' : ['sum','min','max'],
'AMT_CREDIT_SUM_LIMIT': ['sum','min','max'],
'AMT_CREDIT_SUM_OVERDUE':['sum','min','max'],
'DAYS_CREDIT_UPDATE' : ['sum','min','max'],
'AMT_ANNUITY' : ['sum','min','max','mean']
})
 df_bureau_features = df_bureau_features.reset_index()
df_bureau_features.columns = [''.join(col ).strip() for col in df_bureau_features.columns.values]
df_bureau_features_ohe = df_bureau[['SK_ID_CURR'] + ohe_cols].groupby(['SK_ID_CURR'] ).mean().reset_index()
df_bureau_features = pd.merge(df_bureau_features,df_bureau_features_ohe,how = 'left',on = 'SK_ID_CURR')
 del df_bureau_features_ohe
df_train = pd.merge(df_train,df_bureau_features,how = 'left',on = 'SK_ID_CURR')
 df_test = pd.merge(df_test,df_bureau_features,how = 'left',on = 'SK_ID_CURR' )<load_from_csv>


cast_counts = crew_df[['credit_id','imdb_id']].groupby('imdb_id' ).agg('count')
 cast_counts.columns = ['crew_counts']
cast_counts.reset_index(inplace=True)
 df_train= df_train.merge(cast_counts, how='left',left_on='imdb_id',right_on='imdb_id')
 cast_gender_counts = crew_df[['credit_id', 'gender','imdb_id']].groupby(['imdb_id','gender'],as_index=False ).agg('count')
 cast_gender_counts = cast_gender_counts.pivot(index='imdb_id',columns='gender',values='credit_id')
 cast_gender_counts.reset_index(inplace=True)
 cast_gender_counts.fillna(0,inplace=True)
 cast_gender_counts.columns=['imdb_id','gender_crew_count_0','gender_crew_count_1','gender_crew_count_2']
df_train = df_train.merge(cast_gender_counts,how='left',left_on='imdb_id',right_on='imdb_id')
 cast_gender_counts = crew_df[['credit_id', 'gender','imdb_id','department']].groupby(['imdb_id','gender','department'],as_index=False ).agg('count')
 cast_gender_counts = cast_gender_counts.pivot_table(index='imdb_id',columns=['gender','department'],values='credit_id')
 cast_gender_counts.reset_index(inplace=True)
 cast_gender_counts.fillna(0,inplace=True)
 df_train = df_train.merge(cast_gender_counts,how='left',left_on='imdb_id',right_on='imdb_id')
 crew_job_counts = crew_df[['credit_id', 'job','imdb_id']].groupby(['imdb_id','job'],as_index=False ).agg('count')
 crew_job_counts = crew_job_counts.pivot(index='imdb_id',columns='job',values='credit_id')
 crew_job_counts.reset_index(inplace=True)
 crew_job_counts.fillna(0,inplace=True)
 df_train = df_train.merge(crew_job_counts,how='left',left_on='imdb_id',right_on='imdb_id')
 crew_dept_counts = crew_df[['credit_id', 'department','imdb_id']].groupby(['imdb_id','department'],as_index=False ).agg('count')
 crew_dept_counts = crew_dept_counts.pivot(index='imdb_id',columns='department',values='credit_id')
 crew_dept_counts.reset_index(inplace=True)
 crew_dept_counts.fillna(0,inplace=True)
 df_train = df_train.merge(crew_dept_counts,how='left',left_on='imdb_id',right_on='imdb_id')

cast_counts = crew_df_test[['credit_id','imdb_id']].groupby('imdb_id' ).agg('count')
 cast_counts.columns = ['crew_counts']
cast_counts.reset_index(inplace=True)
 df_test= df_test.merge(cast_counts, how='left',left_on='imdb_id',right_on='imdb_id')
 cast_gender_counts = crew_df_test[['credit_id', 'gender','imdb_id']].groupby(['imdb_id','gender'],as_index=False ).agg('count')
 cast_gender_counts = cast_gender_counts.pivot(index='imdb_id',columns='gender',values='credit_id')
 cast_gender_counts.reset_index(inplace=True)
 cast_gender_counts.fillna(0,inplace=True)
 cast_gender_counts.columns=['imdb_id','gender_crew_count_0','gender_crew_count_1','gender_crew_count_2']
df_test = df_test.merge(cast_gender_counts,how='left',left_on='imdb_id',right_on='imdb_id')
 cast_gender_counts = crew_df_test[['credit_id', 'gender','imdb_id','department']].groupby(['imdb_id','gender','department'],as_index=False ).agg('count')
 cast_gender_counts = cast_gender_counts.pivot_table(index='imdb_id',columns=['gender','department'],values='credit_id')
 cast_gender_counts.reset_index(inplace=True)
 cast_gender_counts.fillna(0,inplace=True)
 df_test = df_test.merge(cast_gender_counts,how='left',left_on='imdb_id',right_on='imdb_id')
 crew_job_counts = crew_df_test[['credit_id', 'job','imdb_id']].groupby(['imdb_id','job'],as_index=False ).agg('count')
 crew_job_counts = crew_job_counts.pivot(index='imdb_id',columns='job',values='credit_id')
 crew_job_counts.reset_index(inplace=True)
 crew_job_counts.fillna(0,inplace=True)
 df_test = df_test.merge(crew_job_counts,how='left',left_on='imdb_id',right_on='imdb_id')
 crew_dept_counts = crew_df_test[['credit_id', 'department','imdb_id']].groupby(['imdb_id','department'],as_index=False ).agg('count')
 crew_dept_counts = crew_dept_counts.pivot(index='imdb_id',columns='department',values='credit_id')
 crew_dept_counts.reset_index(inplace=True)
 crew_dept_counts.fillna(0,inplace=True)
 df_test = df_test.merge(crew_dept_counts,how='left',left_on='imdb_id',right_on='imdb_id')
df_previous_application = pd.read_csv(".. /input/previous_application.csv")
 cat_cols = [col for col in df_previous_application.columns if(df_previous_application[col].dtype == object)&(( col != 'SK_ID_CURR')|(col != 'SK_ID_PREV')) ]
df_previous_application = pd.get_dummies(df_previous_application,prefix = cat_cols,columns = cat_cols)
 df_previous_application.head()<load_from_csv>

 character = cast_df['character']
cast = cast_df['name']
character.fillna('x',inplace=True)
 cast.fillna('x',inplace=True)
 character_counts = Counter(character)
 cast_counts = Counter(cast)
 counter_char = {x: character_counts[x] for x in character_counts if character_counts[x] >= 40}
counter_cast = {x: cast_counts[x] for x in cast_counts if cast_counts[x] >= 40}
characters = [x for x in counter_char]
casts = [x for x in counter_cast]
cast_top = cast_df[cast_df['name'].isin(casts)]
cast_top_counts = cast_top[['imdb_id','name','cast_id']].groupby(['imdb_id','name'] ).agg('count')
 cast_top_counts.reset_index(inplace=True)
 char_top = cast_df[cast_df['character'].isin(characters)]
char_top_counts = char_top[['imdb_id','character','cast_id']].groupby(['imdb_id','character'] ).agg('count')
 char_top_counts.reset_index(inplace=True)
 cast_top_test = cast_df_test[cast_df_test['name'].isin(casts)]
cast_top_test_counts = cast_top_test[['imdb_id','name','cast_id']].groupby(['imdb_id','name'] ).agg('count')
 cast_top_test_counts.reset_index(inplace=True)
 char_top_test = cast_df_test[cast_df_test['character'].isin(characters)]
char_top_test_counts = char_top_test[['imdb_id','character','cast_id']].groupby(['imdb_id','character'] ).agg('count')
 char_top_test_counts.reset_index(inplace=True)
 cast_top_counts = cast_top_counts.pivot(index='imdb_id',columns='name',values='cast_id')
 cast_top_counts.fillna(0,inplace=True)
 cast_top_counts.reset_index(inplace=True)
 cast_top_test_counts = cast_top_test_counts.pivot(index='imdb_id',columns='name',values='cast_id')
 cast_top_test_counts.fillna(0,inplace=True)
 cast_top_test_counts.reset_index(inplace=True)
 char_top_counts = char_top_counts.pivot(index='imdb_id',columns='character',values='cast_id')
 char_top_counts.fillna(0,inplace=True)
 char_top_counts.reset_index(inplace=True)
 char_top_test_counts = char_top_test_counts.pivot(index='imdb_id',columns='character',values='cast_id')
 char_top_test_counts.fillna(0,inplace=True)
 char_top_test_counts.reset_index(inplace=True)
 df_train = df_train.merge(char_top_counts,how='left',left_on='imdb_id',right_on='imdb_id')
 df_train = df_train.merge(cast_top_counts,how='left',left_on='imdb_id',right_on='imdb_id')
 df_train.fillna(0,inplace=True)
 df_test = df_test.merge(char_top_test_counts,how='left',left_on='imdb_id',right_on='imdb_id')
 df_test = df_test.merge(cast_top_test_counts,how='left',left_on='imdb_id',right_on='imdb_id')
 df_test.fillna(0,inplace=True )
df_POS_CASH_balance = pd.read_csv(".. /input/POS_CASH_balance.csv")
 df_POS_CASH_balance = pd.get_dummies(df_POS_CASH_balance, columns= ["NAME_CONTRACT_STATUS"])
 df_POS_CASH_balance_current = df_POS_CASH_balance.drop('SK_ID_PREV',axis = 1 ).groupby('SK_ID_CURR' ).mean().reset_index()
df_POS_CASH_balance_previous = df_POS_CASH_balance.drop('SK_ID_CURR',axis = 1 ).groupby('SK_ID_PREV' ).mean().reset_index()
del df_POS_CASH_balance
df_POS_CASH_balance_current.head()<load_from_csv>

 cast = crew_df['name']
cast.fillna('x',inplace=True)
 cast_counts = Counter(cast)
 counter_cast = {x: cast_counts[x] for x in cast_counts if cast_counts[x] >= 40}
casts = [x for x in counter_cast]
cast_top = crew_df[crew_df['name'].isin(casts)]
cast_top_counts = cast_top[['imdb_id','name','credit_id']].groupby(['imdb_id','name'] ).agg('count')
 cast_top_counts.reset_index(inplace=True)
 cast_top_test = crew_df_test[crew_df_test['name'].isin(casts)]
cast_top_test_counts = cast_top_test[['imdb_id','name','credit_id']].groupby(['imdb_id','name'] ).agg('count')
 cast_top_test_counts.reset_index(inplace=True)
 cast_top_counts = cast_top_counts.pivot(index='imdb_id',columns='name',values='credit_id')
 cast_top_counts.fillna(0,inplace=True)
 cast_top_counts.reset_index(inplace=True)
 cast_top_test_counts = cast_top_test_counts.pivot(index='imdb_id',columns='name',values='credit_id')
 cast_top_test_counts.fillna(0,inplace=True)
 cast_top_test_counts.reset_index(inplace=True)
 df_train = df_train.merge(cast_top_counts,how='left',left_on='imdb_id',right_on='imdb_id')
 df_train.fillna(0,inplace=True)
 df_test = df_test.merge(cast_top_test_counts,how='left',left_on='imdb_id',right_on='imdb_id')
 df_test.fillna(0,inplace=True )
df_installments_payments = pd.read_csv(".. /input/installments_payments.csv")
 df_installments_payments_current = df_installments_payments.drop('SK_ID_PREV',axis= 1 ).groupby('SK_ID_CURR' ).mean().reset_index()
df_installments_payments_previous = df_installments_payments.drop('SK_ID_CURR',axis= 1 ).groupby('SK_ID_PREV' ).mean().reset_index()
del df_installments_payments
df_installments_payments_current.head()<load_from_csv>
df_train['overview'].fillna('',inplace=True)
 df_test['overview'].fillna('',inplace=True)
 df_train['len_original_title'] = df_train.original_title.apply(str ).apply(len)
 df_test['len_original_title'] = df_test.original_title.apply(len)
 df_train['len_overview'] = df_train.overview.apply(str ).apply(len)
 df_test['len_overview'] = df_test.overview.apply(str ).apply(len)
 df_train['len_tagline'] = df_train.tagline.apply(str ).apply(len)
 df_test['len_tagline'] = df_test.tagline.apply(str ).apply(len)
 df_train['len_title'] = df_train.title.apply(str ).apply(len)
 df_test['len_title'] = df_test.title.apply(str ).apply(len)
 df_train['num_words_title'] = df_train.title.apply(lambda x: len(str(x ).split(' ')))
 df_test['num_words_title'] = df_test.title.apply(lambda x: len(str(x ).split(' ')))
 df_train['num_words_original_title'] = df_train.original_title.apply(lambda x: len(str(x ).split(' ')))
 df_test['num_words_original_title'] = df_test.original_title.apply(lambda x: len(str(x ).split(' ')))
 df_train['num_words_tagline'] = df_train.tagline.apply(lambda x: len(str(x ).split(' ')))
 df_test['num_words_tagline'] = df_test.tagline.apply(lambda x: len(str(x ).split(' ')))
 df_train['num_words_overview'] = df_train.overview.apply(lambda x: len(str(x ).split(' ')))
 df_test['num_words_overview'] = df_test.overview.apply(lambda x: len(str(x ).split(' ')) )
df_credit_card_balance = pd.read_csv(".. /input/credit_card_balance.csv")
 df_credit_card_balance = pd.get_dummies(df_credit_card_balance, columns= ['NAME_CONTRACT_STATUS'])
 df_credit_card_balance_current = df_credit_card_balance.drop('SK_ID_PREV',axis = 1 ).groupby('SK_ID_CURR' ).mean().reset_index()
df_credit_card_balance_previous = df_credit_card_balance.drop('SK_ID_CURR',axis = 1 ).groupby('SK_ID_PREV' ).mean().reset_index()
del df_credit_card_balance
df_credit_card_balance_current.head()<merge>
def senti(x):
return TextBlob(x ).sentiment
overview_train = df_train['overview']
overview_test = df_test['overview']
tagline_train = df_train['tagline']
tagline_test = df_test['tagline']
overview_train = overview_train.apply(lambda x: " ".join(str(x ).lower() for x in str(x ).split()))
overview_train = overview_train.str.replace('[^\w\s]','')
 overview_test = overview_test.apply(lambda x: " ".join(str(x ).lower() for x in str(x ).split()))
overview_test = overview_test.str.replace('[^\w\s]','')
 tagline_train = tagline_train.apply(lambda x: " ".join(str(x ).lower() for x in str(x ).split()))
tagline_train = tagline_train.str.replace('[^\w\s]','')
 tagline_test = tagline_test.apply(lambda x: " ".join(str(x ).lower() for x in str(x ).split()))
tagline_test = tagline_test.str.replace('[^\w\s]','')
 overview_train = overview_train.apply(lambda x: " ".join(x for x in str(x ).split() if x not in stop))
overview_test = overview_test.apply(lambda x: " ".join(x for x in str(x ).split() if x not in stop))
tagline_train = tagline_train.apply(lambda x: " ".join(x for x in str(x ).split() if x not in stop))
tagline_test = tagline_test.apply(lambda x: " ".join(x for x in str(x ).split() if x not in stop))
st = PorterStemmer()
overview_train = overview_train.apply(lambda x: " ".join([st.stem(word)for word in str(x ).split() ]))
overview_test = overview_test.apply(lambda x: " ".join([st.stem(word)for word in str(x ).split() ]))
tagline_train = tagline_train.apply(lambda x: " ".join([st.stem(word)for word in str(x ).split() ]))
tagline_test = tagline_test.apply(lambda x: " ".join([st.stem(word)for word in str(x ).split() ]))
overview_train_senti = overview_train.apply(senti)
 overview_test_senti = overview_test.apply(senti)
 tagline_train_senti = tagline_train.apply(senti)
 tagline_test_senti = tagline_test.apply(senti)
 df_train['overview_senti_1'] = overview_train_senti.apply(lambda x: x[0])
 df_train['overview_senti_2'] = overview_train_senti.apply(lambda x: x[1])
 df_train['tagline_senti_1'] = tagline_train_senti.apply(lambda x: x[0])
 df_train['tagline_senti_2'] = tagline_train_senti.apply(lambda x: x[1])
 df_test['overview_senti_1'] = overview_test_senti.apply(lambda x: x[0])
 df_test['overview_senti_2'] = overview_test_senti.apply(lambda x: x[1])
 df_test['tagline_senti_1'] = tagline_test_senti.apply(lambda x: x[0])
 df_test['tagline_senti_2'] = tagline_test_senti.apply(lambda x: x[1] )
df_train = df_train.merge(df_POS_CASH_balance_current, on = 'SK_ID_CURR',how = 'left',suffixes=['','_POS_bal_curr'])\ 
.merge(df_installments_payments_current, on = 'SK_ID_CURR',how = 'left', suffixes = ['','_installments_curr'])\ 
.merge(df_credit_card_balance_current, on = 'SK_ID_CURR',how = 'left',suffixes=['','_credit_card_bal_curr'])
 df_test = df_test.merge(df_POS_CASH_balance_current, on = 'SK_ID_CURR',how = 'left',suffixes=['','_POS_bal_curr'])\ 
.merge(df_installments_payments_current, on = 'SK_ID_CURR',how = 'left',suffixes = ['','_installments_curr'])\ 
.merge(df_credit_card_balance_current, on = 'SK_ID_CURR',how = 'left',suffixes=['','_credit_card_bal_curr'])\
df_train.head()<merge>
def process_date(df):
date_parts = ["year", "weekday", "month", 'weekofyear', 'day', 'quarter']
for part in date_parts:
part_col = 'release_date' + "_" + part
df[part_col] = getattr(df['release_date_2'].dt, part ).astype(int)
 return df
def fix_date(x):

 year = x.split('/')[2]
if len(year)== 2:
if int(year)<= 19:
return x[:-2] + '20' + year
else:
return x[:-2] + '19' + year
else:
return x
df_train['release_date_2'] = df_train['release_date'].apply(lambda x: fix_date(x))
df_test['release_date_2'] = df_test['release_date'].apply(lambda x: fix_date(x)if x != 0 else np.NaN)
 df_train['release_date_2'] = pd.to_datetime(df_train['release_date'])
 df_test['release_date_2'] = pd.to_datetime(df_test['release_date'])
 df_train = process_date(df_train)
 df_test = process_date(df_test)
 df_train.drop('release_date_2',axis=1,inplace=True)
 df_test.drop('release_date_2',axis=1,inplace=True )
df_previous_application = df_previous_application.merge(df_POS_CASH_balance_previous, on = 'SK_ID_PREV',how = 'left',suffixes=['','_POS_bal_past'])\ 
.merge(df_installments_payments_previous, on = 'SK_ID_PREV',how = 'left',suffixes = ['','_installments_past'])\ 
.merge(df_credit_card_balance_previous, on = 'SK_ID_PREV',how = 'left',suffixes=['','_credit_card_bal_past'])
 df_previous_application = df_previous_application.drop("SK_ID_PREV",axis= 1 ).groupby(['SK_ID_CURR'] ).mean().reset_index()
df_train = df_train.merge(df_previous_application, on = 'SK_ID_CURR',how = 'left',suffixes=('','_past_appl'))
df_test = df_test.merge(df_previous_application, on = 'SK_ID_CURR',how = 'left',suffixes=('','_past_appl'))
df_previous_application.head()
del df_previous_application<train_model>
train_sub = df_train[['id','collection','release_date_year']]
train_sub = train_sub[train_sub['collection'] != 'x']
train_sub['movie_rank'] = train_sub.groupby('collection')['release_date_year'].rank(ascending=True)
 train_sub.drop('collection',inplace=True,axis=1)
 train_sub.drop('release_date_year',inplace=True,axis=1)
 df_train = df_train.merge(train_sub,how='left',left_on='id',right_on='id')
 test_sub = df_test[['id','collection','release_date_year']]
test_sub = test_sub[test_sub['collection'] != 'x']
test_sub['movie_rank'] = test_sub.groupby('collection')['release_date_year'].rank(ascending=True)
 test_sub.drop('collection',inplace=True,axis=1)
 test_sub.drop('release_date_year',inplace=True,axis=1)
 df_test = df_test.merge(test_sub,how='left',left_on='id',right_on='id')
 df_train.loc[df_train.collection == 'x','movie_rank'] = np.NaN
df_test.loc[df_test.collection == 'x','movie_rank'] = np.NaN
df1 = df_train.sample(frac = 1)
 msk = np.random.rand(len(df1))
eval_set = df1[msk >= 0.95]
train = df1[msk < 0.95]
X_train = train.drop(['TARGET','SK_ID_CURR'],axis = 1)
 Y_train = train['TARGET']
X_eval = eval_set.drop(['TARGET','SK_ID_CURR'],axis = 1)
 Y_eval = eval_set['TARGET']
def xgb_(X_train = X_train, Y_train = Y_train,
params = {
"objective" :['multi:softmax']
,"max_depth" :[2]
,'eta' :[0.1]
},
fit_params = {
'eval_metric' :['mlogloss']
,'eval_set' :[(X_eval,Y_eval)]},
X_eval = X_eval, Y_eval = Y_eval):
print("
print()
model_xgb = xgb.XGBClassifier()
clf = GridSearchCV(model_xgb, param_grid = params,
fit_params = fit_params,cv =3, scoring ="roc_auc")
 clf.fit(X_train, Y_train)
 print("
Best parameters set found on development set:")
 print()
print(clf.best_params_)
 print()
print("Grid scores on development set:")
 print()
means = clf.cv_results_['mean_test_score']
stds = clf.cv_results_['std_test_score']
for mean, std, params in zip(means, stds, clf.cv_results_['params']):
print("%0.3f(+/-%0.03f)for %r"
%(mean, std * 2, params))
print()
print("Detailed classification report:")
 print()
print("The model is trained on the full development set.")
 print("The scores are computed on the full evaluation set.")
 print()
y_true, y_pred = Y_eval, clf.predict(X_eval)
 print(classification_report(y_true, y_pred))
print()
print(confusion_matrix(y_true, y_pred))
print()
model_xgb = clf.best_estimator_
xgb.plot_importance(model_xgb,max_num_features = 20)
 Y_dev_pred = model_xgb.predict_proba(X_eval)[:,1]
score = roc_auc_score(Y_eval,Y_dev_pred)
 results = model_xgb.evals_result()
metrics = fit_params['eval_metric']
epochs = len(results['validation_0'][metrics[0]])
 x_axis = range(0, epochs)
 for metric in metrics:
fig, ax = pyplot.subplots()
ax.plot(x_axis, results['validation_0'][metric], label='Train')
 ax.plot(x_axis, results['validation_1'][metric], label='Validation/Hold out set')
 ax.legend()
pyplot.ylabel('%s' %(metric))
pyplot.title('XGBoost %s' %(metric))
pyplot.show()
return model_xgb,score,clf,results,model_xgb.feature_importances_ 
<train_model>
le = LabelEncoder()
le2 = LabelEncoder()
le3 = LabelEncoder()
merged_collection = pd.concat([df_train.collection,df_test.collection])
 merged_language = merged_df['original_language']
le.fit(merged_collection)
 le2.fit(merged_language)
 train_collection = df_train['collection']
test_collection = df_test['collection']
train_language = df_train['original_language']
test_language = df_test['original_language']
train_collection_labels = le.transform(train_collection)
 test_collection_labels = le.transform(test_collection)
 train_language_labels = le2.transform(train_language)
 test_language_labels = le2.transform(test_language)
 df_train['collection_encoded'] = train_collection_labels
df_test['collection_encoded'] = test_collection_labels
df_train['original_language_encoded'] = train_language_labels
df_test['original_language_encoded'] = test_language_labels
xgb_params = {
'learning_rate' :[0.02]
,'reg_lambda' :[16]
,"max_depth" :[9]
,'silent' :[False]
,'n_estimators' :[1000]
,'colsample_bytree' :[0.5]
,'nthread' :[-1]
,'subsample' :[0.5]
,'objective' :["binary:logistic"]
,'scale_pos_weight' :[2]
}
fit_params = {
'eval_metric' :['auc']
,'eval_set' :[(X_train,Y_train),(X_eval,Y_eval)]
,'early_stopping_rounds' : 30
}
model_xgb,score_xgb,xgb_gridsearch,results,feats = xgb_(X_train = X_train,X_eval = X_eval,params = xgb_params,fit_params = fit_params)
<predict_on_test>
stop_english = stopwords.words('english')
 ls_all_words = []
ls_all_words_tagline = []
count_words_param = 30

 corpus_overview = df_train['overview']
corpus_overview_list = corpus_overview.tolist()
corpus_overview = [str(re.sub(' +',' ',str(txt)))for txt in corpus_overview_list]
corpus_overview = [str(re.sub('[^A-Za-z0-9' ]+','',str(txt)))for txt in corpus_overview]
corpus_overview = [str(txt ).lower() for txt in corpus_overview]
bagofwords = [collections.Counter(re.findall(r'\w+',txt)) for txt in corpus_overview]
sumbags = sum(bagofwords,collections.Counter())
 sumbags_filter = {x : sumbags[x] for x in sumbags if sumbags[x] >= count_words_param}
for ele in sumbags_filter:
if(ele not in stop_english and ele not in ls_all_words):
ls_all_words.append(ele)
 list_index = list(range(len(df_train)))
 df = pd.DataFrame(0, index=list_index, columns=ls_all_words)
 i = 0
for item in bagofwords:
for ele in item:
if(ele in ls_all_words):
df.set_value(i,ele,item[ele])
 i += 1
df_train = pd.concat([df_train.reset_index(drop=True),df],axis=1)
 corpus_overview = df_train['tagline']
corpus_overview_list = corpus_overview.tolist()
corpus_overview = [str(re.sub(' +',' ',str(txt)))for txt in corpus_overview_list]
corpus_overview = [str(re.sub('[^A-Za-z0-9' ]+','',str(txt)))for txt in corpus_overview]
corpus_overview = [str(txt ).lower() for txt in corpus_overview]
bagofwords = [collections.Counter(re.findall(r'\w+',txt)) for txt in corpus_overview]
sumbags = sum(bagofwords,collections.Counter())
 sumbags_filter = {x : sumbags[x] for x in sumbags if sumbags[x] >= count_words_param}
for ele in sumbags_filter:
if(ele not in stop_english and ele not in ls_all_words_tagline):
ls_all_words_tagline.append(ele)
 list_index = list(range(len(df_train)))
 df = pd.DataFrame(0, index=list_index, columns=ls_all_words_tagline)
 i = 0
for item in bagofwords:
for ele in item:
if(ele in ls_all_words_tagline):
df.set_value(i,ele,item[ele])
 i += 1
df_train = pd.concat([df_train.reset_index(drop=True),df],axis=1)

corpus_overview_test = df_test['overview']
corpus_overview_test_list = corpus_overview_test.tolist()
corpus_overview_test = [str(re.sub(' +',' ',str(txt)))for txt in corpus_overview_test_list]
corpus_overview_test = [str(re.sub('[^A-Za-z0-9' ]+','',str(txt)))for txt in corpus_overview_test]
corpus_overview_test = [str(txt ).lower() for txt in corpus_overview_test]
bagofwords = [collections.Counter(re.findall(r'\w+',txt)) for txt in corpus_overview_test]
sumbags = sum(bagofwords,collections.Counter())
 sumbags_filter = {x : sumbags[x] for x in sumbags if sumbags[x] >= count_words_param}
list_index = list(range(len(df_test)))
 df = pd.DataFrame(0, index=list_index, columns=ls_all_words)
 i = 0
for item in bagofwords:
for ele in item:
if(ele in ls_all_words):
df.set_value(i,ele,item[ele])
 i += 1
df_test = pd.concat([df_test.reset_index(drop=True),df],axis=1)
 corpus_overview_test = df_test['tagline']
corpus_overview_test_list = corpus_overview_test.tolist()
corpus_overview_test = [str(re.sub(' +',' ',str(txt)))for txt in corpus_overview_test_list]
corpus_overview_test = [str(re.sub('[^A-Za-z0-9' ]+','',str(txt)))for txt in corpus_overview_test]
corpus_overview_test = [str(txt ).lower() for txt in corpus_overview_test]
bagofwords = [collections.Counter(re.findall(r'\w+',txt)) for txt in corpus_overview_test]
sumbags = sum(bagofwords,collections.Counter())
 sumbags_filter = {x : sumbags[x] for x in sumbags if sumbags[x] >= count_words_param}
list_index = list(range(len(df_test)))
 df = pd.DataFrame(0, index=list_index, columns=ls_all_words_tagline)
 i = 0
for item in bagofwords:
for ele in item:
if(ele in ls_all_words_tagline):
df.set_value(i,ele,item[ele])
 i += 1
df_test = pd.concat([df_test.reset_index(drop=True),df],axis=1 )
X_test = df_test.drop(['TARGET','SK_ID_CURR'],axis = 1)
 Y_test = model_xgb.predict_proba(X_test)[:,1]<save_to_csv>
df_test.loc[df_test['id'] == 3033,'budget'] = 250
df_test.loc[df_test['id'] == 3051,'budget'] = 50
df_test.loc[df_test['id'] == 3084,'budget'] = 337
df_test.loc[df_test['id'] == 3224,'budget'] = 4
df_test.loc[df_test['id'] == 3594,'budget'] = 25
df_test.loc[df_test['id'] == 3619,'budget'] = 500
df_test.loc[df_test['id'] == 3831,'budget'] = 3
df_test.loc[df_test['id'] == 3935,'budget'] = 500
df_test.loc[df_test['id'] == 4049,'budget'] = 995946
df_test.loc[df_test['id'] == 4424,'budget'] = 3
df_test.loc[df_test['id'] == 4460,'budget'] = 8
df_test.loc[df_test['id'] == 4555,'budget'] = 1200000
df_test.loc[df_test['id'] == 4624,'budget'] = 30
df_test.loc[df_test['id'] == 4645,'budget'] = 500
df_test.loc[df_test['id'] == 4709,'budget'] = 450
df_test.loc[df_test['id'] == 4839,'budget'] = 7
df_test.loc[df_test['id'] == 3125,'budget'] = 25
df_test.loc[df_test['id'] == 3142,'budget'] = 1
df_test.loc[df_test['id'] == 3201,'budget'] = 450
df_test.loc[df_test['id'] == 3222,'budget'] = 6
df_test.loc[df_test['id'] == 3545,'budget'] = 38
df_test.loc[df_test['id'] == 3670,'budget'] = 18
df_test.loc[df_test['id'] == 3792,'budget'] = 19
df_test.loc[df_test['id'] == 3881,'budget'] = 7
df_test.loc[df_test['id'] == 3969,'budget'] = 400
df_test.loc[df_test['id'] == 4196,'budget'] = 6
df_test.loc[df_test['id'] == 4221,'budget'] = 11
df_test.loc[df_test['id'] == 4222,'budget'] = 500
df_test.loc[df_test['id'] == 4285,'budget'] = 11
df_test.loc[df_test['id'] == 4319,'budget'] = 1
df_test.loc[df_test['id'] == 4639,'budget'] = 10
df_test.loc[df_test['id'] == 4719,'budget'] = 45
df_test.loc[df_test['id'] == 4822,'budget'] = 22
df_test.loc[df_test['id'] == 4829,'budget'] = 20
df_test.loc[df_test['id'] == 4969,'budget'] = 20
df_test.loc[df_test['id'] == 5021,'budget'] = 40
df_test.loc[df_test['id'] == 5035,'budget'] = 1
df_test.loc[df_test['id'] == 5063,'budget'] = 14
df_test.loc[df_test['id'] == 5119,'budget'] = 2
df_test.loc[df_test['id'] == 5214,'budget'] = 30
df_test.loc[df_test['id'] == 5221,'budget'] = 50
df_test.loc[df_test['id'] == 4903,'budget'] = 15
df_test.loc[df_test['id'] == 4983,'budget'] = 3
df_test.loc[df_test['id'] == 5102,'budget'] = 28
df_test.loc[df_test['id'] == 5217,'budget'] = 75
df_test.loc[df_test['id'] == 5224,'budget'] = 3
df_test.loc[df_test['id'] == 5469,'budget'] = 20
df_test.loc[df_test['id'] == 5840,'budget'] = 1
df_test.loc[df_test['id'] == 5960,'budget'] = 30
df_test.loc[df_test['id'] == 6506,'budget'] = 11
df_test.loc[df_test['id'] == 6553,'budget'] = 280
df_test.loc[df_test['id'] == 6561,'budget'] = 7
df_test.loc[df_test['id'] == 6582,'budget'] = 218
df_test.loc[df_test['id'] == 6638,'budget'] = 5
df_test.loc[df_test['id'] == 6749,'budget'] = 8
df_test.loc[df_test['id'] == 6759,'budget'] = 50
df_test.loc[df_test['id'] == 6856,'budget'] = 10
df_test.loc[df_test['id'] == 6858,'budget'] = 100
df_test.loc[df_test['id'] == 6876,'budget'] = 250
df_test.loc[df_test['id'] == 6972,'budget'] = 1
df_test.loc[df_test['id'] == 7079,'budget'] = 8000000
df_test.loc[df_test['id'] == 7150,'budget'] = 118
df_test.loc[df_test['id'] == 6506,'budget'] = 118
df_test.loc[df_test['id'] == 7225,'budget'] = 6
df_test.loc[df_test['id'] == 7231,'budget'] = 85
df_test.loc[df_test['id'] == 5222,'budget'] = 5
df_test.loc[df_test['id'] == 5322,'budget'] = 90
df_test.loc[df_test['id'] == 5350,'budget'] = 70
df_test.loc[df_test['id'] == 5378,'budget'] = 10
df_test.loc[df_test['id'] == 5545,'budget'] = 80
df_test.loc[df_test['id'] == 5810,'budget'] = 8
df_test.loc[df_test['id'] == 5926,'budget'] = 300
df_test.loc[df_test['id'] == 5927,'budget'] = 4
df_test.loc[df_test['id'] == 5986,'budget'] = 1
df_test.loc[df_test['id'] == 6053,'budget'] = 20
df_test.loc[df_test['id'] == 6104,'budget'] = 1
df_test.loc[df_test['id'] == 6130,'budget'] = 30
df_test.loc[df_test['id'] == 6301,'budget'] = 150
df_test.loc[df_test['id'] == 6276,'budget'] = 100
df_test.loc[df_test['id'] == 6473,'budget'] = 100
df_test.loc[df_test['id'] == 6842,'budget'] = 30
df_train.loc[df_train['id'] == 90,'budget'] = 30000000
df_train.loc[df_train['id'] == 118,'budget'] = 60000000
df_train.loc[df_train['id'] == 149,'budget'] = 18000000
df_train.loc[df_train['id'] == 464,'budget'] = 20000000
df_train.loc[df_train['id'] == 470,'budget'] = 13000000
df_train.loc[df_train['id'] == 513,'budget'] = 930000
df_train.loc[df_train['id'] == 797,'budget'] = 8000000
df_train.loc[df_train['id'] == 819,'budget'] = 90000000
df_train.loc[df_train['id'] == 850,'budget'] = 90000000
df_train.loc[df_train['id'] == 1007,'budget'] = 2
df_train.loc[df_train['id'] == 1112,'budget'] = 7500000
df_train.loc[df_train['id'] == 1131,'budget'] = 4300000
df_train.loc[df_train['id'] == 1359,'budget'] = 10000000
df_train.loc[df_train['id'] == 1542,'budget'] = 1
df_train.loc[df_train['id'] == 1570,'budget'] = 15800000
df_train.loc[df_train['id'] == 1571,'budget'] = 4000000
df_train.loc[df_train['id'] == 1714,'budget'] = 46000000
df_train.loc[df_train['id'] == 1721,'budget'] = 17500000
df_train.loc[df_train['id'] == 1885,'budget'] = 12
df_train.loc[df_train['id'] == 2091,'budget'] = 10
df_train.loc[df_train['id'] == 2268,'budget'] = 17500000
df_train.loc[df_train['id'] == 2491,'budget'] = 6
df_train.loc[df_train['id'] == 2602,'budget'] = 31000000
df_train.loc[df_train['id'] == 2612,'budget'] = 15000000
df_train.loc[df_train['id'] == 2696,'budget'] = 10000000
df_train.loc[df_train['id'] == 2801,'budget'] = 10000000
df_train.loc[df_train['id'] == 335,'budget'] = 2
df_train.loc[df_train['id'] == 348,'budget'] = 12
df_train.loc[df_train['id'] == 470,'budget'] = 13000000
df_train.loc[df_train['id'] == 513,'budget'] = 1100000
df_train.loc[df_train['id'] == 640,'budget'] = 6
df_train.loc[df_train['id'] == 696,'budget'] = 1
df_train.loc[df_train['id'] == 797,'budget'] = 8000000
df_train.loc[df_train['id'] == 850,'budget'] = 1500000
df_train.loc[df_train['id'] == 1199,'budget'] = 5
df_train.loc[df_train['id'] == 1282,'budget'] = 9
df_train.loc[df_train['id'] == 1347,'budget'] = 1
df_train.loc[df_train['id'] == 1755,'budget'] = 2
df_train.loc[df_train['id'] == 1801,'budget'] = 5
df_train.loc[df_train['id'] == 1918,'budget'] = 592
df_train.loc[df_train['id'] == 2033,'budget'] = 4
df_train.loc[df_train['id'] == 2118,'budget'] = 344
df_train.loc[df_train['id'] == 2252,'budget'] = 130
df_train.loc[df_train['id'] == 2256,'budget'] = 1
df_train.loc[df_train['id'] == 2696,'budget'] = 10000000
df_application_test["TARGET"] = Y_test
df_submit = df_application_test[["SK_ID_CURR","TARGET"]]
df_submit.to_csv('submission_appl_bureau.csv', index=False )<set_options>

 train_add_features = pd.read_csv('.. /input/tmdb-additional-features/TrainAdditionalFeatures.csv')
 test_add_features = pd.read_csv('.. /input/tmdb-additional-features/TestAdditionalFeatures.csv')
 df_train = df_train.merge(train_add_features, how='left',left_on='imdb_id',right_on='imdb_id')
 df_test = df_test.merge(test_add_features, how='left',left_on='imdb_id',right_on='imdb_id')
warnings.filterwarnings('ignore')
 gc.enable()<load_from_csv>
train_data_cols = df_train.columns
test_data_cols = df_test.columns
for col_test in test_data_cols:
if str(col_test)not in train_data_cols and str(col_test)!= 'year_bucket' and not str(col_test ).__contains__('0')and not str(col_test ).__contains__('1')and not str(col_test ).__contains__('2'):
print('dropping ' + str(col_test))
df_test.drop(str(col_test),axis=1,inplace=True)
 for col_test in train_data_cols:
if str(col_test)not in test_data_cols and str(col_test)!= 'year_bucket' and not str(col_test ).__contains__('0')and not str(col_test ).__contains__('1')and not str(col_test ).__contains__('2'):
if(str(col_test)== 'revenue'):
continue
print('dropping ' + str(col_test))
df_train.drop(str(col_test),axis=1,inplace=True )
print('Importing data...')
 lgbm_submission = pd.read_csv('.. /input/sample_submission.csv' )<categorify>
def is_collection(row):
if(row == 0):
return 0
else:
return 1
def is_homepage(row):
if(row == 0):
return 0
else:
return 1
df_train['is_collection'] = df_train.belongs_to_collection.apply(is_collection)
 df_train['is_homepage'] = df_train.homepage.apply(is_homepage)
 df_test['is_collection'] = df_test.belongs_to_collection.apply(is_collection)
 df_test['is_homepage'] = df_test.homepage.apply(is_homepage)
 df_train['has_production_comp'] = df_train.production_companies.apply(is_collection)
 df_test['has_production_comp'] = df_test.production_companies.apply(is_collection)
 df_train['has_production_count'] = df_train.production_countries.apply(is_collection)
 df_test['has_production_count'] = df_test.production_countries.apply(is_collection)
 df_train['has_keywords'] = df_train.Keywords.apply(is_collection)
 df_test['has_keywords'] = df_test.Keywords.apply(is_collection)
 df_train['has_tagline'] = df_train.tagline.apply(is_collection)
 df_test['has_tagline'] = df_test.tagline.apply(is_collection )
buro_bal = pd.read_csv('.. /input/bureau_balance.csv')
 print('Buro bal shape : ', buro_bal.shape)
 print('transform to dummies')
 buro_bal = pd.concat([buro_bal, pd.get_dummies(buro_bal.STATUS, prefix='buro_bal_status')], axis=1 ).drop('STATUS', axis=1)
 print('Counting buros')
 buro_counts = buro_bal[['SK_ID_BUREAU', 'MONTHS_BALANCE']].groupby('SK_ID_BUREAU' ).count()
buro_bal['buro_count'] = buro_bal['SK_ID_BUREAU'].map(buro_counts['MONTHS_BALANCE'])
 print('averaging buro bal')
 avg_buro_bal = buro_bal.groupby('SK_ID_BUREAU' ).mean()
avg_buro_bal.columns = ['avg_buro_' + f_ for f_ in avg_buro_bal.columns]
del buro_bal
gc.collect()
print('Read Bureau')
 buro = pd.read_csv('.. /input/bureau.csv')
 print('Go to dummies')
 buro_credit_active_dum = pd.get_dummies(buro.CREDIT_ACTIVE, prefix='ca_')
 buro_credit_currency_dum = pd.get_dummies(buro.CREDIT_CURRENCY, prefix='cu_')
 buro_credit_type_dum = pd.get_dummies(buro.CREDIT_TYPE, prefix='ty_')
 buro_full = pd.concat([buro, buro_credit_active_dum, buro_credit_currency_dum, buro_credit_type_dum], axis=1)
 del buro_credit_active_dum, buro_credit_currency_dum, buro_credit_type_dum
gc.collect()
print('Merge with buro avg')
 buro_full = buro_full.merge(right=avg_buro_bal.reset_index() , how='left', on='SK_ID_BUREAU', suffixes=('', '_bur_bal'))
print('Counting buro per SK_ID_CURR')
 nb_bureau_per_curr = buro_full[['SK_ID_CURR', 'SK_ID_BUREAU']].groupby('SK_ID_CURR' ).count()
buro_full['SK_ID_BUREAU'] = buro_full['SK_ID_CURR'].map(nb_bureau_per_curr['SK_ID_BUREAU'])
 print('Averaging bureau')
 avg_buro = buro_full.groupby('SK_ID_CURR' ).mean()
del buro, buro_full
gc.collect()
print('Read prev')
 prev = pd.read_csv('.. /input/previous_application.csv')
 prev_cat_features = [
f_ for f_ in prev.columns if prev[f_].dtype == 'object'
]
print('Go to dummies')
 prev_dum = pd.DataFrame()
for f_ in prev_cat_features:
prev_dum = pd.concat([prev_dum, pd.get_dummies(prev[f_], prefix=f_ ).astype(np.uint8)], axis=1)
 prev = pd.concat([prev, prev_dum], axis=1)
 del prev_dum
gc.collect()
print('Counting number of Prevs')
 nb_prev_per_curr = prev[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
prev['SK_ID_PREV'] = prev['SK_ID_CURR'].map(nb_prev_per_curr['SK_ID_PREV'])
 print('Averaging prev')
 avg_prev = prev.groupby('SK_ID_CURR' ).mean()
del prev
gc.collect()
print('Reading POS_CASH')
 pos = pd.read_csv('.. /input/POS_CASH_balance.csv')
 print('Go to dummies')
 pos = pd.concat([pos, pd.get_dummies(pos['NAME_CONTRACT_STATUS'])], axis=1)
 print('Compute nb of prevs per curr')
 nb_prevs = pos[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
pos['SK_ID_PREV'] = pos['SK_ID_CURR'].map(nb_prevs['SK_ID_PREV'])
 print('Go to averages')
 avg_pos = pos.groupby('SK_ID_CURR' ).mean()
del pos, nb_prevs
gc.collect()
print('Reading CC balance')
 cc_bal = pd.read_csv('.. /input/credit_card_balance.csv')
 print('Go to dummies')
 cc_bal = pd.concat([cc_bal, pd.get_dummies(cc_bal['NAME_CONTRACT_STATUS'], prefix='cc_bal_status_')], axis=1)
 nb_prevs = cc_bal[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
cc_bal['SK_ID_PREV'] = cc_bal['SK_ID_CURR'].map(nb_prevs['SK_ID_PREV'])
 print('Compute average')
 avg_cc_bal = cc_bal.groupby('SK_ID_CURR' ).mean()
avg_cc_bal.columns = ['cc_bal_' + f_ for f_ in avg_cc_bal.columns]
del cc_bal, nb_prevs
gc.collect()
print('Reading Installments')
 inst = pd.read_csv('.. /input/installments_payments.csv')
 nb_prevs = inst[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
inst['SK_ID_PREV'] = inst['SK_ID_CURR'].map(nb_prevs['SK_ID_PREV'])
 avg_inst = inst.groupby('SK_ID_CURR' ).mean()
avg_inst.columns = ['inst_' + f_ for f_ in avg_inst.columns]
print('Read data and test')
 data = pd.read_csv('.. /input/application_train.csv')
 test = pd.read_csv('.. /input/application_test.csv')
 print('Shapes : ', data.shape, test.shape)
 y = data['TARGET']
del data['TARGET']
categorical_feats = [
f for f in data.columns if data[f].dtype == 'object'
]
categorical_feats
for f_ in categorical_feats:
data[f_], indexer = pd.factorize(data[f_])
 test[f_] = indexer.get_indexer(test[f_])
 data = data.merge(right=avg_buro.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_buro.reset_index() , how='left', on='SK_ID_CURR')
 data = data.merge(right=avg_prev.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_prev.reset_index() , how='left', on='SK_ID_CURR')
 data = data.merge(right=avg_pos.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_pos.reset_index() , how='left', on='SK_ID_CURR')
 data = data.merge(right=avg_cc_bal.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_cc_bal.reset_index() , how='left', on='SK_ID_CURR')
 data = data.merge(right=avg_inst.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_inst.reset_index() , how='left', on='SK_ID_CURR')
 del avg_buro, avg_prev
gc.collect()<split>
df_train.drop('belongs_to_collection',axis=1,inplace=True)
 df_train.drop('collection',axis=1,inplace=True)
 df_train.drop('genres',axis=1,inplace=True)
 df_train.drop('homepage',axis=1,inplace=True)
 df_train.drop('imdb_id',axis=1,inplace=True)
 df_train.drop('original_language',axis=1,inplace=True)
 df_train.drop('original_title',axis=1,inplace=True)
 df_train.drop('overview',axis=1,inplace=True)
 df_train.drop('poster_path',axis=1,inplace=True)
 df_train.drop('production_companies',axis=1,inplace=True)
 df_train.drop('production_countries',axis=1,inplace=True)
 df_train.drop('release_date',axis=1,inplace=True)
 df_train.drop('spoken_languages',axis=1,inplace=True)
 df_train.drop('status',axis=1,inplace=True)
 df_train.drop('tagline',axis=1,inplace=True)
 df_train.drop('title',axis=1,inplace=True)
 df_train.drop('Keywords',axis=1,inplace=True)
 df_train.drop('cast',axis=1,inplace=True)
 df_train.drop('crew',axis=1,inplace=True)
 df_test.drop('belongs_to_collection',axis=1,inplace=True)
 df_test.drop('collection',axis=1,inplace=True)
 df_test.drop('genres',axis=1,inplace=True)
 df_test.drop('homepage',axis=1,inplace=True)
 df_test.drop('imdb_id',axis=1,inplace=True)
 df_test.drop('original_language',axis=1,inplace=True)
 df_test.drop('original_title',axis=1,inplace=True)
 df_test.drop('overview',axis=1,inplace=True)
 df_test.drop('poster_path',axis=1,inplace=True)
 df_test.drop('production_companies',axis=1,inplace=True)
 df_test.drop('production_countries',axis=1,inplace=True)
 df_test.drop('release_date',axis=1,inplace=True)
 df_test.drop('spoken_languages',axis=1,inplace=True)
 df_test.drop('status',axis=1,inplace=True)
 df_test.drop('tagline',axis=1,inplace=True)
 df_test.drop('title',axis=1,inplace=True)
 df_test.drop('Keywords',axis=1,inplace=True)
 df_test.drop('cast',axis=1,inplace=True)
 df_test.drop('crew',axis=1,inplace=True )
train_x, valid_x, train_y, valid_y = train_test_split(data, y, test_size=0.2, shuffle=True, stratify=y, random_state=1301 )<create_dataframe>
df_train['revenue'].fillna(0,inplace=True)
 df_train['log_revenue'] = np.log(df_train['revenue'])
 df_train['budget'].fillna(0,inplace=True)
 df_train['log_budget'] = np.log(df_train['budget'])
 df_train.drop('revenue',axis=1,inplace=True)
 df_train.drop('budget',axis=1,inplace=True)
 df_test['budget'].fillna(0,inplace=True)
 df_test['log_budget'] = np.log(df_test['budget'])
 df_test.drop('budget',axis=1,inplace=True)
 df_train['budget_year'] = df_train['log_budget'] / df_train['release_date_year']
df_test['budget_year'] = df_test['log_budget'] / df_test['release_date_year']
df_train['popularity_year'] = df_train['popularity'] / df_train['release_date_year']
df_test['popularity_year'] = df_test['popularity'] / df_test['release_date_year']
df_train.runtime.fillna(0,inplace=True)
 df_test.runtime.fillna(0,inplace=True)
 bins = [0, 40, 50, 60, 70, 80,90, 100,110,120,130,140,150,160,170,180,190,200,210,220,230,240,250,260,270,280,290,300,310,320,330,340,350]
labels = [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32]
df_train['binned_runtime'] = pd.cut(df_train['runtime'], bins=bins, labels=labels)
 df_test['binned_runtime'] = pd.cut(df_test['runtime'], bins=bins, labels=labels)
 df_train['binned_runtime'] = pd.to_numeric(df_train['binned_runtime'])
 df_test['binned_runtime'] = pd.to_numeric(df_test['binned_runtime'] )
train_data=lgb.Dataset(train_x,label=train_y)
 valid_data=lgb.Dataset(valid_x,label=valid_y )<init_hyperparams>
num_rounds = 10000
y_label = df_train['log_revenue']
X = df_train.drop('log_revenue',axis=1)
 X.drop('id',axis=1,inplace=True)
 X['cast_1_gender'] = X['cast_1_gender'].apply(int)
 X['cast_2_gender'] = X['cast_2_gender'].apply(int)
 X['cast_3_gender'] = X['cast_3_gender'].apply(int)
 X = X.replace([np.inf, -np.inf], 0 ).fillna(0)
 X.fillna(0,inplace=True)
 X_test = df_test.copy()
X_test.fillna(0,inplace=True)
 X_test_id = df_test['id']
X_test.drop('id',axis=1,inplace=True)
 X_test['cast_1_gender'] = X_test['cast_1_gender'].apply(int)
 X_test['cast_2_gender'] = X_test['cast_2_gender'].apply(int)
 X_test['cast_3_gender'] = X_test['cast_3_gender'].apply(int)
 X_test = X_test.replace([np.inf, -np.inf], 0 ).fillna(0)
 X_test.fillna(0,inplace=True)
 X = X.loc[:,~X.columns.duplicated() ]
X_test = X_test.loc[:,~X_test.columns.duplicated() ]
dtrain = xgb.DMatrix(data=X,label=y_label)
 dtest = xgb.DMatrix(data=X_test)
 watchlist = [(dtest, 'eval'),(dtrain, 'train')]
param = {'eta':0.01, 'max_depth': 2, 'booster': 'gbtree', 'colsample_bytree' : 0.3,'subsample' : 0.9}
res = xgb.cv(param,dtrain,num_rounds,10,metrics={'rmse'})
 print('cross validation')
 print(res)
 bst = xgb.train(param,dtrain,num_rounds)
 y_pred = bst.predict(dtrain)
 print('Train rmsle')
 print(mean_squared_error(y_label, y_pred)** 0.5)
 y_pred_reg = bst.predict(dtest)
 y_pred_reg_exp = np.exp(y_pred_reg)
 y_pred_reg_exp = pd.DataFrame(y_pred_reg_exp)
 test_preds = pd.concat([X_test_id,y_pred_reg_exp],axis=1)
 test_preds.columns = ['id','revenue']
test_preds.to_csv('submission_xgb.csv',index=False )