Kaggle competition: TMDB Box Office Prediction
id_ = ['Id', 'idhogar', 'Target']<define_variables>
pd.set_option('max_columns', None)
 %matplotlib inline
plt.style.use('ggplot')
 stop = set(stopwords.words('english'))
py.init_notebook_mode(connected=True)
ind_bool = ['v18q', 'dis', 'male', 'female', 'estadocivil1', 'estadocivil2', 'estadocivil3',
'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco1', 'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5',
'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12', 'instlevel1', 'instlevel2', 'instlevel3',
'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8',
'instlevel9', 'mobilephone', 'rez_esc-missing']
ind_ordered = ['rez_esc', 'escolari', 'age']<define_variables>
or_train = pd.read_csv('.. /input/tmdb-box-office-prediction/train.csv')
 or_test = pd.read_csv('.. /input/tmdb-box-office-prediction/test.csv')
 dict_columns = ['belongs_to_collection', 'genres', 'production_companies',
'production_countries', 'spoken_languages', 'Keywords', 'cast', 'crew']
def text_to_dict(df):
for column in dict_columns:
df[column] = df[column].apply(lambda x: {} if pd.isna(x)else ast.literal_eval(x))
return df
or_train = text_to_dict(or_train)
 or_test = text_to_dict(or_test)
 train = or_train.copy()
test = or_test.copy()
hh_bool = ['hacdor', 'hacapo', 'v14a', 'refrig', 'paredblolad', 'paredzocalo',
'paredpreb','pisocemento', 'pareddes', 'paredmad',
'paredzinc', 'paredfibras', 'paredother', 'pisomoscer', 'pisoother',
'pisonatur', 'pisonotiene', 'pisomadera',
'techozinc', 'techoentrepiso', 'techocane', 'techootro', 'cielorazo',
'abastaguadentro', 'abastaguafuera', 'abastaguano',
'public', 'planpri', 'noelec', 'coopele', 'sanitario1',
'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6',
'energcocinar1', 'energcocinar2', 'energcocinar3', 'energcocinar4',
'elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4',
'elimbasu5', 'elimbasu6', 'epared1', 'epared2', 'epared3',
'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3',
'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5',
'computer', 'television', 'lugar1', 'lugar2', 'lugar3',
'lugar4', 'lugar5', 'lugar6', 'area1', 'area2', 'v2a1-missing']
hh_ordered = [ 'rooms', 'r4h1', 'r4h2', 'r4h3', 'r4m1','r4m2','r4m3', 'r4t1', 'r4t2',
'r4t3', 'v18q1', 'tamhog','tamviv','hhsize','hogar_nin',
'hogar_adul','hogar_mayor','hogar_total', 'bedrooms', 'qmobilephone']
hh_cont = ['v2a1', 'dependency', 'edjefe', 'edjefa', 'meaneduc', 'overcrowding']<define_variables>
chs = ['belongs_to_collection']
for table in [train, test]:
for col in chs:
trs = table[col].apply(lambda x:sorted(list(map(lambda y:y['name'], x))))
text = trs.apply(lambda x:" ".join(x))
table[col] = text
sqr_ = ['SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe',
'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned', 'agesq']<define_variables>
text_train = or_train.copy()
text_test = or_test.copy()
text_train=text_train.fillna('')
 text_test=text_test.fillna('')
 text_cols = ['belongs_to_collection','genres','production_companies','production_countries','Keywords','cast','crew']
for col in text_cols:
trs=text_train[col].apply(lambda x:sorted(list(map(lambda y:y['name'], x))))
text = trs.apply(lambda x:" ".join(x)+" " if len(x)>0 else "")
 text_train[col] = text.apply(lambda x:x.strip())
 trs=text_test[col].apply(lambda x:sorted(list(map(lambda y:y['name'], x))))
text = trs.apply(lambda x:" ".join(x)+" " if len(x)>0 else "")
 text_test[col] = text.apply(lambda x:x.strip() )
x= ind_bool + ind_ordered + id_ + hh_bool + hh_ordered + hh_cont + sqr_
print('There are no repeats : ', np.all(np.array(list(Counter(x ).values())) == 1))
print('We covered every variable : ', len(x)== data.shape[1])
<define_variables>
choose_text = ['belongs_to_collection','genres','production_companies','production_countries','overview']
text_train['text'] = text_train[choose_text].apply(lambda x: ''.join(x), axis=1)
 text_train = text_train['text']
text_train = text_train.apply(lambda x:x.split(' '))
text_test['text'] = text_test[choose_text].apply(lambda x: ''.join(x), axis=1)
 text_test = text_test['text']
text_test = text_test.apply(lambda x:x.split(' '))
heads = data.loc[data['parentesco1'] == 1, :]
heads = heads[id_ + hh_bool+ hh_cont + hh_ordered]
heads.shape<data_type_conversions>
embeddings_index = {}
with open(os.path.join('.. /input/glove6b50dtxt/glove.6B.50d.txt')) as f:
for line in f:
values = line.split()
word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
corr_matrix = heads.corr()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(abs(upper[column])> 0.95)]
to_drop 
<filter>
EMBEDDING_DIM = max(max(text_train.apply(lambda x:len(x))), max(text_test.apply(lambda x:len(x)))) *50
num_words = len(text_train)
 embedding_matrix = np.zeros(( num_words, EMBEDDING_DIM))
test_matrix = np.zeros(( len(text_test), EMBEDDING_DIM))
tamhog_corr = corr_matrix.loc[corr_matrix['tamhog'].abs() > 0.9, corr_matrix['tamhog'].abs() > 0.9]
tamhog_corr<drop_column>
for row in range(len(text_train)) :
line = text_train[row]
for i in range(len(line)) :
k = line[i].lower()
if k in embeddings_index:
vec = embeddings_index[k]
embedding_matrix[row][i*50:(i+1)*50] = vec
for row in range(len(text_test)) :
line = text_test[row]
for i in range(len(line)) :
k = line[i].lower()
if k in embeddings_index:
vec = embeddings_index[k]
test_matrix[row][i*50:(i+1)*50] = vec
heads = heads.drop(columns = ['tamhog', 'hogar_total', 'r4t3'] )<data_type_conversions>
input_img = Input(shape=(EMBEDDING_DIM,), name='input_img')
 encode1 = Dense(EMBEDDING_DIM//2, activation='relu', name='encode1' )(input_img)
 hidden = Dropout(0.5 )(encode1)
 encode2 = Dense(128, activation='relu', name='encode2' )(hidden)
 encode3 = Dense(32, activation='relu', name='encode3' )(encode2)
 bottleneck = Dense(8, activation='relu', name='bottleneck' )(encode3)
 decode1 = Dense(32, activation='relu', name='decode1' )(bottleneck)
 decode2 = Dense(128, activation='relu', name='decode2' )(decode1)
 decode3 = Dense(EMBEDDING_DIM//2, activation='relu', name='decode3' )(decode2)
 hidden = Dropout(0.5 )(decode3)
 decode4 = Dense(EMBEDDING_DIM, activation='relu', name='decode4' )(decode3)
 ae = models.Model(input_img, decode4 )
corr_matrix = heads.corr()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(abs(upper[column])> 0.95)]
to_drop<filter>
learning_rate = 1E-3
ae.compile(loss='mean_squared_error', optimizer=optimizers.RMSprop(lr=learning_rate))
history = ae.fit(x_tr, x_tr,
batch_size=128,
epochs=20,
validation_data=(x_val, x_val))
corr_matrix.loc[corr_matrix['coopele'].abs() > 0.9, corr_matrix['coopele'].abs() > 0.9]<feature_engineering>
ae_encoder = models.Model(input_img, bottleneck)
 encoded_test = ae_encoder.predict(test_matrix)
 encoded_test
encoded_train = ae_encoder.predict(embedding_matrix)
 encoded_train
elec = []
for i , row in heads.iterrows() :
if row['noelec'] == 1:
elec.append(0)
 elif row['coopele'] == 1:
elec.append(1)
 elif row['public'] == 1:
elec.append(2)
 elif row['planpri'] == 1:
elec.append(3)
 else:
elec.append(np.nan)
 heads['elec'] = elec
heads['elec-missing'] = heads['elec'].isnull() 
<groupby>
xg_train = xg.DMatrix(data=train.loc[:,['budget','popularity','runtime']], label=np.array(train['revenue']))
params = {'eval_metric' : 'rmse', 'silent' : 1}
xg_model = xg.train(params, xg_train)
 xg_test = xg.DMatrix(data=train.loc[:,['budget', 'popularity', 'runtime']])
 xg_pred = xg_model.predict(xg_test)
 eli5.show_weights(xg_model )
heads = heads.drop(columns = 'area2')
 heads.groupby('area1')['Target'].value_counts(normalize = True )<drop_column>
print(np.sqrt(metrics.mean_squared_log_error(train['revenue'], li_pre)))
 print(np.sqrt(metrics.mean_squared_log_error(train['revenue'], xg_pred)) )
heads['roof'] = np.argmax(np.array(heads[['etecho1', 'etecho2', 'etecho3']]), axis = 1)
 heads = heads.drop(columns = ['etecho1', 'etecho2', 'etecho3'])
 heads['floor'] = np.argmax(np.array(heads[['eviv1', 'eviv2', 'eviv3']]), axis = 1)
<prepare_output>
train['log_budget'] = np.log1p(train['budget'])
 train['log_revenue'] = np.log1p(train['revenue'])
 test['log_budget'] = np.log1p(test['budget'] )
counts = pd.DataFrame(heads.groupby(['walls+roof+floor'])['Target'].value_counts(normalize=True)).rename(columns={'Target' : 'Normalized Count'} ).reset_index()
counts.head()<feature_engineering>
for table in [train, test]:
table['has_homepage'] = 0
table.loc[table['homepage'].isnull() == True, 'has_homepage'] = 0
table.loc[table['homepage'].isnull() == False, 'has_homepage'] = 1
heads['warning'] = 1 *(heads['sanitario1'] + 
(heads['elec'] == 0)+
heads['pisonotiene'] +
heads['abastaguano'] + 
(heads['cielorazo'] == 0))<feature_engineering>
def fix_date(x):
year = x.split('/')[2]
if int(year)<= 19:
return x[:-2]+"20"+year
else:
return x[:-2]+"19"+year
test.loc[test['release_date'].isnull() , 'release_date'] = '5/1/00'
for table in [train, test]:
table['release_date'] = table['release_date'].apply(lambda x:fix_date(x))
heads['phones-per-capita'] = heads['qmobilephone'] / heads['tamviv']
heads['tablets-per-capita'] = heads['v18q1'] / heads['tamviv']
heads['rooms-per-capita'] = heads['rooms'] / heads['tamviv']
heads['rent-per-capita'] = heads['v2a1'] / heads['tamviv']<import_modules>
def trans(table):
df = pd.DataFrame()
t = pd.to_datetime(table['release_date'])
 date_parts = ["year", "weekday", "month", 'weekofyear', 'day', 'quarter']
for part in date_parts:
part_col = 'release_date' + "_" + part
df[part_col] = getattr(t.dt, part ).astype(int)
 table = pd.concat([table, df], axis=1)
 return table
train = trans(train)
 test = trans(test )
from scipy.stats import spearmanr<create_dataframe>
train = train.drop(['homepage', 'imdb_id', 'poster_path', 'release_date', 'status','log_revenue'], axis=1)
 test = test.drop(['homepage', 'imdb_id', 'poster_path', 'release_date','status'], axis=1 )
warnings.filterwarnings('ignore', category = RuntimeWarning)
 feats = []
scorr = []
pvalues = []
for c in heads:
if heads[c].dtype != 'object':
feats.append(c)
 scorr.append(spearmanr(train_heads[c], train_heads['Target'] ).correlation)
 pvalues.append(spearmanr(train_heads[c], train_heads['Target'] ).pvalue)
 scorrs = pd.DataFrame({'feature': feats , 'scorr' : scorr, 'pvalue' : pvalues} ).sort_values('scorr')
<merge>
train.rename(columns={'belongs_to_collection':"collection_name"}, inplace=True)
 test.rename(columns={'belongs_to_collection':"collection_name"}, inplace=True )
corrs = pcorrs.merge(scorrs, on='feature')
 corrs['diff'] = corrs['pcorr'] - corrs['scorr']
corrs.sort_values('diff' ).head()<categorify>
train['log_budget'] = np.log1p(train['budget'])
 test['log_budget'] = np.log1p(test['budget'] )
train_heads['meaneduc'].fillna(train_heads['meaneduc'].mean() , inplace=True )<data_type_conversions>
list_of_genres = list(train['genres'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 train['num_genres'] = train['genres'].apply(lambda x: len(x)if x != {} else 0)
 train['all_genres'] = train['genres'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_genres = [m[0] for m in Counter([i for j in list_of_genres for i in j] ).most_common(15)]
for g in top_genres:
train['genre_' + g] = train['all_genres'].apply(lambda x: 1 if g in x else 0)
 test['num_genres'] = test['genres'].apply(lambda x: len(x)if x != {} else 0)
 test['all_genres'] = test['genres'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_genres:
test['genre_' + g] = test['all_genres'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['genres'], axis=1)
 test = test.drop(['genres'], axis=1 )
corr_matrix = ind.corr()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(abs(upper[column])> 0.95)]
to_drop<drop_column>
list_of_companies = list(train['production_companies'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 train['num_companies'] = train['production_companies'].apply(lambda x: len(x)if x != {} else 0)
 train['all_production_companies'] = train['production_companies'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_companies = [m[0] for m in Counter([i for j in list_of_companies for i in j] ).most_common(30)]
for g in top_companies:
train['production_company_' + g] = train['all_production_companies'].apply(lambda x: 1 if g in x else 0)
 test['num_companies'] = test['production_companies'].apply(lambda x: len(x)if x != {} else 0)
 test['all_production_companies'] = test['production_companies'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_companies:
test['production_company_' + g] = test['all_production_companies'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['production_companies', 'all_production_companies'], axis=1)
 test = test.drop(['production_companies', 'all_production_companies'], axis=1 )
ind = ind.drop(columns = 'male' )<groupby>
list_of_countries = list(train['production_countries'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 train['num_countries'] = train['production_countries'].apply(lambda x: len(x)if x != {} else 0)
 train['all_countries'] = train['production_countries'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_countries = [m[0] for m in Counter([i for j in list_of_countries for i in j] ).most_common(25)]
for g in top_countries:
train['production_country_' + g] = train['all_countries'].apply(lambda x: 1 if g in x else 0)
 test['num_countries'] = test['production_countries'].apply(lambda x: len(x)if x != {} else 0)
 test['all_countries'] = test['production_countries'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_countries:
test['production_country_' + g] = test['all_countries'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['production_countries', 'all_countries'], axis=1)
 test = test.drop(['production_countries', 'all_countries'], axis=1 )
range_ = lambda x : x.max() - x.min()
range_.__name__ = 'range_'
ind_agg = ind.drop(columns = 'Target' ).groupby('idhogar' ).agg(['min', 'max', 'sum', 'count', 'std', range_])
 ind_agg.head()<rename_columns>
list_of_languages = list(train['spoken_languages'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 train['num_languages'] = train['spoken_languages'].apply(lambda x: len(x)if x != {} else 0)
 train['all_languages'] = train['spoken_languages'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_languages = [m[0] for m in Counter([i for j in list_of_languages for i in j] ).most_common(30)]
for g in top_languages:
train['language_' + g] = train['all_languages'].apply(lambda x: 1 if g in x else 0)
 test['num_languages'] = test['spoken_languages'].apply(lambda x: len(x)if x != {} else 0)
 test['all_languages'] = test['spoken_languages'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_languages:
test['language_' + g] = test['all_languages'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['spoken_languages', 'all_languages'], axis=1)
 test = test.drop(['spoken_languages', 'all_languages'], axis=1 )
new_col = []
for c in ind_agg.columns.levels[0]:
for stat in ind_agg.columns.levels[1]:
new_col.append(f'{c}-{stat}')
 ind_agg.columns = new_col
ind_agg.head()<merge>
list_of_keywords = list(train['Keywords'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 train['num_Keywords'] = train['Keywords'].apply(lambda x: len(x)if x != {} else 0)
 train['all_Keywords'] = train['Keywords'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_keywords = [m[0] for m in Counter([i for j in list_of_keywords for i in j] ).most_common(30)]
for g in top_keywords:
train['keyword_' + g] = train['all_Keywords'].apply(lambda x: 1 if g in x else 0)
 test['num_Keywords'] = test['Keywords'].apply(lambda x: len(x)if x != {} else 0)
 test['all_Keywords'] = test['Keywords'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_keywords:
test['keyword_' + g] = test['all_Keywords'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['Keywords', 'all_Keywords'], axis=1)
 test = test.drop(['Keywords', 'all_Keywords'], axis=1 )
ind_agg = ind_agg.drop(columns = to_drop)
 ind_feats = list(ind_agg.columns)
 final = heads.merge(ind_agg, on ='idhogar', how = 'left')
 print('Rinal features shape : ', final.shape )<merge>
list_of_cast_names = list(train['cast'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 list_of_cast_genders = list(train['cast'].apply(lambda x: [i['gender'] for i in x] if x != {} else [] ).values)
 list_of_cast_characters = list(train['cast'].apply(lambda x: [i['character'] for i in x] if x != {} else [] ).values)
 train['num_cast'] = train['cast'].apply(lambda x: len(x)if x != {} else 0)
 top_cast_names = [m[0] for m in Counter([i for j in list_of_cast_names for i in j] ).most_common(15)]
for g in top_cast_names:
train['cast_name_' + g] = train['cast'].apply(lambda x: 1 if g in str(x)else 0)
 train['genders_0_cast'] = train['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
train['genders_1_cast'] = train['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
train['genders_2_cast'] = train['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
top_cast_characters = [m[0] for m in Counter([i for j in list_of_cast_characters for i in j] ).most_common(15)]
for g in top_cast_characters:
train['cast_character_' + g] = train['cast'].apply(lambda x: 1 if g in str(x)else 0)
 test['num_cast'] = test['cast'].apply(lambda x: len(x)if x != {} else 0)
 for g in top_cast_names:
test['cast_name_' + g] = test['cast'].apply(lambda x: 1 if g in str(x)else 0)
 test['genders_0_cast'] = test['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
test['genders_1_cast'] = test['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
test['genders_2_cast'] = test['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
for g in top_cast_characters:
test['cast_character_' + g] = test['cast'].apply(lambda x: 1 if g in str(x)else 0)
 train = train.drop(['cast'], axis=1)
 test = test.drop(['cast'], axis=1 )
head_gender = ind.loc[ind['parentesco1'] == 1 , ['idhogar', 'female']]
final = final.merge(head_gender, on = 'idhogar', how='left' ).rename(columns={'female' : 'female-head'} )<count_values>
list_of_crew_names = list(train['crew'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 list_of_crew_jobs = list(train['crew'].apply(lambda x: [i['job'] for i in x] if x != {} else [] ).values)
 list_of_crew_genders = list(train['crew'].apply(lambda x: [i['gender'] for i in x] if x != {} else [] ).values)
 list_of_crew_departments = list(train['crew'].apply(lambda x: [i['department'] for i in x] if x != {} else [] ).values)
 train['num_crew'] = train['crew'].apply(lambda x: len(x)if x != {} else 0)
 top_crew_names = [m[0] for m in Counter([i for j in list_of_crew_names for i in j] ).most_common(15)]
for g in top_crew_names:
train['crew_name_' + g] = train['crew'].apply(lambda x: 1 if g in str(x)else 0)
 train['genders_0_crew'] = train['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
train['genders_1_crew'] = train['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
train['genders_2_crew'] = train['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
top_cast_characters = [m[0] for m in Counter([i for j in list_of_cast_characters for i in j] ).most_common(15)]
for g in top_cast_characters:
train['crew_character_' + g] = train['crew'].apply(lambda x: 1 if g in str(x)else 0)
 top_crew_jobs = [m[0] for m in Counter([i for j in list_of_crew_jobs for i in j] ).most_common(15)]
for j in top_crew_jobs:
train['jobs_' + j] = train['crew'].apply(lambda x: sum([1 for i in x if i['job'] == j]))
top_crew_departments = [m[0] for m in Counter([i for j in list_of_crew_departments for i in j] ).most_common(15)]
for j in top_crew_departments:
train['departments_' + j] = train['crew'].apply(lambda x: sum([1 for i in x if i['department'] == j]))
test['num_crew'] = test['crew'].apply(lambda x: len(x)if x != {} else 0)
 for g in top_crew_names:
test['crew_name_' + g] = test['crew'].apply(lambda x: 1 if g in str(x)else 0)
 test['genders_0_crew'] = test['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
test['genders_1_crew'] = test['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
test['genders_2_crew'] = test['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
for g in top_cast_characters:
test['crew_character_' + g] = test['crew'].apply(lambda x: 1 if g in str(x)else 0)
 for j in top_crew_jobs:
test['jobs_' + j] = test['crew'].apply(lambda x: sum([1 for i in x if i['job'] == j]))
for j in top_crew_departments:
test['departments_' + j] = test['crew'].apply(lambda x: sum([1 for i in x if i['department'] == j]))
train = train.drop(['crew'], axis=1)
 test = test.drop(['crew'], axis=1 )
final.groupby('female-head')['Target'].value_counts(normalize=True )<groupby>
for table in [test, train]:
table['has_collection'] = table['collection_name'].apply(lambda x: 1 if len(x)!=0 else 0 )
final.groupby('female-head')['meaneduc'].agg(['mean', 'count'] )<import_modules>
for col in ['original_language', 'collection_name', 'all_genres']:
le = LabelEncoder()
le.fit(list(train[col].fillna('')) + list(test[col].fillna('')))
 train[col] = le.transform(train[col].fillna('' ).astype(str))
test[col] = le.transform(test[col].fillna('' ).astype(str))
scorer = make_scorer(f1_score, greater_is_better=True, average='macro' )<data_type_conversions>
train_texts = train[['title', 'tagline', 'overview', 'original_title']]
test_texts = test[['title', 'tagline', 'overview', 'original_title']]
train_labels = np.array(list(final[final['Target'].notnull() ]['Target'].astype(np.uint8)))
 train_set = final[final['Target'].notnull() ].drop(columns = ['Id', 'idhogar', 'Target'])
 test_set = final[final['Target'].isnull() ].drop(columns = ['Id', 'idhogar', 'Target'])
 submission_base = test[['Id', 'idhogar']].copy()<categorify>
for col in ['title', 'tagline', 'overview', 'original_title']:
train['len_' + col] = train[col].fillna('' ).apply(lambda x: len(str(x)))
 train['words_' + col] = train[col].fillna('' ).apply(lambda x: len(str(x.split(' '))))
train = train.drop(col, axis=1)
 test['len_' + col] = test[col].fillna('' ).apply(lambda x: len(str(x)))
 test['words_' + col] = test[col].fillna('' ).apply(lambda x: len(str(x.split(' '))))
test = test.drop(col, axis=1 )
features = list(train_set.columns)
 pipeline = Pipeline([('imputer', Imputer(strategy = 'median')) , 
('scaler', MinMaxScaler())])
 train_set = pipeline.fit_transform(train_set)
 test_set = pipeline.transform(test_set )<compute_train_metric>
X = train.drop(['id', 'revenue'], axis=1)
 y = np.log1p(train['revenue'])
 X_test = test.drop(['id'], axis=1)
 X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.1 )
model = RandomForestClassifier(n_estimators = 100, random_state=10, n_jobs= -1)
 cv_score = cross_val_score(model, train_set, train_labels, cv=10 , scoring = scorer)
 print(f'10 Fold Cross Validation F1 Score= {round(cv_score.mean() , 4)} with std = {round(cv_score.std() , 4)}' )<train_model>
params = {'num_leaves': 30,
'min_data_in_leaf': 20,
'objective': 'regression',
'max_depth': 5,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.2,
"verbosity": -1}
model1 = lgb.LGBMRegressor(**params, n_estimators = 20000, nthread = 4, n_jobs = -1)
 model1.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='rmse',
verbose=1000, early_stopping_rounds=200)
 eli5.show_weights(model1, feature_filter=lambda x: x != '<BIAS>' )
model.fit(train_set, train_labels)
 feature_importances = pd.DataFrame({'feature' : features, 'importance':model.feature_importances_})
 feature_importances.head()<import_modules>
n_fold = 5
folds = KFold(n_splits=n_fold, shuffle=True, random_state=42 )
from sklearn.svm import LinearSVC
from sklearn.naive_bayes import GaussianNB
from sklearn.neural_network import MLPClassifier
from sklearn.linear_model import LogisticRegressionCV, RidgeClassifierCV
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.neighbors import KNeighborsClassifier<compute_train_metric>
def train_model(X, X_test, y, params=None, folds=folds, model_type='lgb', plot_feature_importance=False, model=None):
oof = np.zeros(X.shape[0])
 prediction = np.zeros(X_test.shape[0])
 scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(folds.split(X)) :
print('Fold', fold_n, 'started at', time.ctime())
 if model_type == 'sklearn':
X_train, X_valid = X[train_index], X[valid_index]
else:
X_train, X_valid = X.values[train_index], X.values[valid_index]
y_train, y_valid = y[train_index], y[valid_index]
if model_type == 'lgb':
model = lgb.LGBMRegressor(**params, n_estimators = 20000, nthread = 4, n_jobs = -1)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='rmse',
verbose=1000, early_stopping_rounds=200)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration_)
 if model_type == 'xgb':
train_data = xgb.DMatrix(data=X_train, label=y_train)
 valid_data = xgb.DMatrix(data=X_valid, label=y_valid)
 watchlist = [(train_data, 'train'),(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=20000, evals=watchlist, early_stopping_rounds=200, verbose_eval=500, params=params)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid), ntree_limit=model.best_ntree_limit)
 y_pred = model.predict(xgb.DMatrix(X_test.values), ntree_limit=model.best_ntree_limit)
 if model_type == 'sklearn':
model = model
model.fit(X_train, y_train)
 y_pred_valid = model.predict(X_valid ).reshape(-1,)
 score = mean_squared_error(y_valid, y_pred_valid)
 y_pred = model.predict(X_test)
 if model_type == 'cat':
model = CatBoostRegressor(iterations=20000, eval_metric='RMSE', **params)
 model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=False)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test)
 oof[valid_index] = y_pred_valid.reshape(-1,)
 scores.append(mean_squared_error(y_valid, y_pred_valid)** 0.5)
 prediction += y_pred
if model_type == 'lgb':
fold_importance = pd.DataFrame()
fold_importance["feature"] = X.columns
fold_importance["importance"] = model.feature_importances_
fold_importance["fold"] = fold_n + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 prediction /= n_fold
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 if model_type == 'lgb':
feature_importance["importance"] /= n_fold
if plot_feature_importance:
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
return oof, prediction, feature_importance
return oof, prediction
else:
return oof, prediction
model_results = cv_model(train_set, train_labels, LinearSVC() , 'LSVC', model_results )<compute_train_metric>
params = {'num_leaves': 30,
'min_data_in_leaf': 10,
'objective': 'regression',
'max_depth': 5,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.2,
"verbosity": -1}
oof_lgb, prediction_lgb, _ = train_model(X, X_test, y, params=params, model_type='lgb', plot_feature_importance=True )
model_results = cv_model(train_set, train_labels, GaussianNB() , 'GNB', model_results )<create_dataframe>
df2 = pd.DataFrame(encoded_train)
 X=pd.concat([X, df2], axis=1)
 df2 = pd.DataFrame(encoded_test)
 X_test = pd.concat([X_test, df2], axis=1)
 X
model_results = cv_model(train_set, train_labels, MLPClassifier(hidden_layer_sizes =(32,64,128,64,32)) , 'MLP', model_results )<compute_train_metric>
xg_model = XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bytree=1, gamma=4.541003990662603, importance_type='gain',
learning_rate=0.08209238500752991, max_delta_step=0, max_depth=4,
min_child_weight=8, missing=None, n_estimators=137, n_jobs=1,
nthread=None, objective='reg:linear', random_state=0, reg_alpha=0,
reg_lambda=1, scale_pos_weight=1, seed=None, silent=True,
subsample=1)
 xg_train = xg.DMatrix(data=X, label=np.array(y))
xg_model.fit(X, y)
 xg_test = xg.DMatrix(data=X_test)
 xg_pred = xg_model.predict(X_test )
model_results = cv_model(train_set, train_labels, LinearDiscriminantAnalysis() , 'LDA', model_results )<compute_train_metric>
train_pred = xg_model.predict(X)
 train_pred = np.expm1(train_pred)
 np.sqrt(mean_squared_log_error(train['revenue'], train_pred))
model_results = cv_model(train_set, train_labels, RidgeClassifierCV() , 'RIDGE', model_results )<find_best_params>
sub = pd.read_csv('.. /input/tmdb-box-office-prediction/sample_submission.csv')
 sub['revenue'] = np.expm1(xg_pred)
 sub.to_csv("xgb.csv", index=False )
for n in [5,10,20]:
print(f'
KNN with {n} neighbors
')
 model_results = cv_model(train_set, train_labels,
KNeighborsClassifier(n_neighbors = n),
f'knn-{n}', model_results )<choose_model_class>
from sklearn.model_selection import RandomizedSearchCV, GridSearchCV
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import StratifiedKFold
from xgboost import XGBRegressor
model_results = cv_model(train_set, train_labels ,
ExtraTreesClassifier(n_estimators = 100 , random_state = 10),
'EXT', model_results )<compute_train_metric>
xgb = XGBRegressor()
baseline = cross_val_score(xgb, X, y, scoring='neg_mean_squared_error' ).mean()
baseline
<filter><EOS> 
param_dist = {"learning_rate": uniform(0, 1),
"gamma": uniform(0, 5),
"max_depth": range(1,50),
"n_estimators": range(1,300),
"min_child_weight": range(1,10)}
rs = RandomizedSearchCV(xgb, param_distributions=param_dist,
scoring='neg_mean_squared_error', n_iter=25)
 rs.fit(X, y);