Kaggle competition: TMDB Box Office Prediction
<SOS> metric: RMSLE Kaggle data source: tmdb-box-office-prediction<set_options>
color = sns.color_palette()
sns.set_style('darkgrid')
 %matplotlib inline
%config InlineBackend.figure_format = 'retina'
pd.set_option('display.float_format', lambda x: '{:.3f}'.format(x))
pd.set_option('max_columns', None)
 print(os.listdir(".. /input/"))
gc.collect()<predict_on_test>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.head()
train_model4 = model4.fit(X_train, y_train)
 pred4 = train_model4.predict(X_test)
 print("Accuracy for model 4: %.2f" %(accuracy_score(y_test, pred4)* 100)) 
<compute_test_metric>
dict_columns = ['belongs_to_collection', 'genres', 'production_companies',
'production_countries', 'spoken_languages', 'Keywords', 'cast', 'crew']
def text_to_dict(df):
for column in dict_columns:
df[column] = df[column].apply(lambda x: {} if pd.isna(x)else ast.literal_eval(x))
return df
train = text_to_dict(train)
 test = text_to_dict(test )
print('Model 4 XGboost Report %r' %(classification_report(y_test, pred4)) )<compute_test_metric>
for i, e in enumerate(train['belongs_to_collection'][:5]):
print(i, e )
confusion_matrix(y_test, pred4 )<load_from_csv>
train['belongs_to_collection'].apply(lambda x: len(x)if x != {} else 0 ).value_counts()
df_test = pd.read_csv('.. /input/test.csv' )<feature_engineering>
train['collection_name'] = train['belongs_to_collection'].apply(lambda x: x[0]['name'] if x != {} else 0)
 train['has_collection'] = train['belongs_to_collection'].apply(lambda x: len(x)if x != {} else 0)
 test['collection_name'] = test['belongs_to_collection'].apply(lambda x: x[0]['name'] if x != {} else 0)
 test['has_collection'] = test['belongs_to_collection'].apply(lambda x: len(x)if x != {} else 0)
 train = train.drop(['belongs_to_collection'], axis=1)
 test = test.drop(['belongs_to_collection'], axis=1 )
df_test.loc[df_test.edjefa == 'yes', 'edjefa'] = 1
df_test.loc[df_test.edjefa == 'no', 'edjefa'] = 0
df_test.loc[df_test.edjefe == 'yes', 'edjefe'] = 1
df_test.loc[df_test.edjefe == 'no', 'edjefe'] = 0
df_test[['edjefa','edjefe']] = df_test[['edjefa','edjefe']].apply(pd.to_numeric)
 df_test[['edjefa','edjefe']].dtypes<prepare_x_and_y>
for i, e in enumerate(train['genres'][:5]):
print(i, e )
X_actual_test = df_test[total_features]<predict_on_test>
print('Number of genres in films')
 train['genres'].apply(lambda x: len(x)if x != {} else 0 ).value_counts()
pred_actual = train_model6.predict(X_actual_test)
 pred_actual<create_dataframe>
list_of_genres = list(train['genres'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values )
df_final = pd.DataFrame(df['Id'], pred_actual ).reset_index()
df_final.columns = ['Target','Id'] 
<define_variables>
train['num_genres'] = train['genres'].apply(lambda x: len(x)if x != {} else 0)
 train['all_genres'] = train['genres'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_genres = [m[0] for m in Counter([i for j in list_of_genres for i in j] ).most_common(15)]
for g in top_genres:
train['genre_' + g] = train['all_genres'].apply(lambda x: 1 if g in x else 0)
 test['num_genres'] = test['genres'].apply(lambda x: len(x)if x != {} else 0)
 test['all_genres'] = test['genres'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_genres:
test['genre_' + g] = test['all_genres'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['genres'], axis=1)
 test = test.drop(['genres'], axis=1 )
cols = cols[-1:] + cols[:-1]
cols 
<rename_columns>
for i, e in enumerate(train['production_companies'][:5]):
print(i, e )
df_final.index.name = None
df_final.head(7 )<count_values>
print('Number of production companies in films')
 train['production_companies'].apply(lambda x: len(x)if x != {} else 0 ).value_counts()
df_final['Target'].value_counts() 
<save_to_csv>
list_of_companies = list(train['production_companies'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values )
df_final[cols].to_csv('sample_submission.csv', index=False )<set_options>
train['num_companies'] = train['production_companies'].apply(lambda x: len(x)if x != {} else 0)
 train['all_production_companies'] = train['production_companies'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_companies = [m[0] for m in Counter([i for j in list_of_companies for i in j] ).most_common(30)]
for g in top_companies:
train['production_company_' + g] = train['all_production_companies'].apply(lambda x: 1 if g in x else 0)
 test['num_companies'] = test['production_companies'].apply(lambda x: len(x)if x != {} else 0)
 test['all_production_companies'] = test['production_companies'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_companies:
test['production_company_' + g] = test['all_production_companies'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['production_companies', 'all_production_companies'], axis=1)
 test = test.drop(['production_companies', 'all_production_companies'], axis=1 )
%matplotlib inline<load_from_csv>
for i, e in enumerate(train['production_countries'][:5]):
print(i, e )
data_path = '.. /input'
train_data = pd.read_csv(os.path.join(data_path, 'train.csv'))
train_data.head(3 )<define_variables>
print('Number of production countries in films')
 train['production_countries'].apply(lambda x: len(x)if x != {} else 0 ).value_counts()
tablets = []
for owns, num_tablets in zip(train_data['v18q'], train_data['v18q1']):
if owns == 0:
tablets += [0]
else:
tablets += [num_tablets]
train_data['v18q1'] = tablets<define_variables>
list_of_countries = list(train['production_countries'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_countries for i in j] ).most_common(25 )
tmp_educ = []
sq_tmp_educ = []
for efe, efa, meduc, sq in zip(train_data['edjefe'], train_data['edjefa'], train_data['meaneduc'], train_data['SQBmeaned']):
new_educ = meduc
if new_educ != new_educ:
if efa == "no":
if efe == "no":
new_educ = 0.0
else:
new_educ = float(efe)
 else:
if efe == "no":
new_educ = float(efa)
 else:
new_educ = float(efe)+ float(efa)
 if meduc != new_educ:
print(meduc, ",", new_educ)
 tmp_educ += [new_educ]
sq_tmp_educ += [new_educ ** 2]
train_data['meaneduc'] = tmp_educ
train_data['SQBmeaned'] = sq_tmp_educ<feature_engineering>
train['num_countries'] = train['production_countries'].apply(lambda x: len(x)if x != {} else 0)
 train['all_countries'] = train['production_countries'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_countries = [m[0] for m in Counter([i for j in list_of_countries for i in j] ).most_common(25)]
for g in top_countries:
train['production_country_' + g] = train['all_countries'].apply(lambda x: 1 if g in x else 0)
 test['num_countries'] = test['production_countries'].apply(lambda x: len(x)if x != {} else 0)
 test['all_countries'] = test['production_countries'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_countries:
test['production_country_' + g] = test['all_countries'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['production_countries', 'all_countries'], axis=1)
 test = test.drop(['production_countries', 'all_countries'], axis=1 )
v2a1 = []
rez_esc = []
for rentpay, rez in zip(train_data['v2a1'], train_data['rez_esc']):
if rentpay != rentpay:
v2a1 += [0]
else:
v2a1 += [rentpay]
if rez != rez:
rez_esc += [0]
else:
rez_esc += [rez]
train_data['rez_esc'] = rez_esc<feature_engineering>
for i, e in enumerate(train['spoken_languages'][:5]):
print(i, e )
depend = []
for dependency, children, olds, total in zip(train_data['dependency'], train_data['hogar_nin'], train_data['hogar_mayor'], train_data['hogar_total']):
calc_depend = False
if depend != depend:
calc_depend = True
elif(dependency == "yes" or dependency == "no"):
calc_depend = True
if calc_depend:
i =(children + olds)/(total - children - olds)
 else:
i = float(dependency)
 depend += [i]
train_data['dependency'] = depend
chw = []
for nin, adul in zip(train_data['hogar_nin'], train_data['hogar_adul']):
if adul == 0:
chw += [nin * 2]
else:
chw += [nin / adul]
train_data['child_weight'] =(train_data['hogar_nin'] + train_data['hogar_mayor'])/ train_data['hogar_total']
train_data['child_weight2'] = chw
train_data['child_weight3'] = train_data['r4t1'] / train_data['r4t3']
train_data['work_power'] = train_data['dependency'] * train_data['hogar_adul']
train_data['SQBworker'] = train_data['hogar_adul'] ** 2
train_data['rooms_per_person'] = train_data['rooms'] /(train_data['tamviv'])
 train_data['bedrooms_per_room'] = train_data['bedrooms'] / train_data['rooms']
train_data['female_weight'] = train_data['r4m3'] / train_data['r4t3']<normalization>
print('Number of spoken languages in films')
 train['spoken_languages'].apply(lambda x: len(x)if x != {} else 0 ).value_counts()
v2a1_drop = ['edjefe', 'edjefa', 'v2a1', 'Id']
train_hh = train_data.query('parentesco1 == 1' ).drop('Target', axis=1)
 v2a1_train_tmp = train_hh.query('v2a1 == v2a1')
 v2a1_train = v2a1_train_tmp.drop(v2a1_drop, axis=1)
 v2a1_train = v2a1_train.drop('idhogar', axis=1)
 v2a1_train_target = v2a1_train_tmp['v2a1'].copy()
v2a1_test_tmp = train_hh.query('v2a1 != v2a1' ).drop(v2a1_drop, axis=1)
 v2a1_test = v2a1_test_tmp.drop('idhogar', axis=1)
 forest = GradientBoostingRegressor(n_estimators=400, learning_rate=0.2, max_depth=5, random_state=0)
 forest.fit(v2a1_train, v2a1_train_target)
 print("score: ", forest.score(v2a1_train, v2a1_train_target))
v2a1_train_pred = forest.predict(v2a1_train)
 forest_mse = mean_squared_error(v2a1_train_target, v2a1_train_pred)
 print("RMSE: ", np.sqrt(forest_mse))<predict_on_test>
list_of_languages = list(train['spoken_languages'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_languages for i in j] ).most_common(15 )
v2a1_pred = pd.DataFrame({'ID': v2a1_test_tmp['idhogar'], 'v2a1_pred': forest.predict(v2a1_test)})
 v2a1 = []
for rent, idhh in zip(train_data['v2a1'], train_data['idhogar']):
if rent != rent:
i = 0
for rent_p, index in zip(v2a1_pred['v2a1_pred'], v2a1_pred['ID']):
if index == idhh:
i = rent_p
break
if i == 0:
for rent_org, index in zip(v2a1_train_target, v2a1_train_tmp['idhogar']):
if index == idhh:
i = rent_org
if i < 0:
i = 0
v2a1 += [i]
else:
v2a1 += [rent]
train_data['v2a1'] = v2a1<count_values>
train['num_languages'] = train['spoken_languages'].apply(lambda x: len(x)if x != {} else 0)
 train['all_languages'] = train['spoken_languages'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_languages = [m[0] for m in Counter([i for j in list_of_languages for i in j] ).most_common(30)]
for g in top_languages:
train['language_' + g] = train['all_languages'].apply(lambda x: 1 if g in x else 0)
 test['num_languages'] = test['spoken_languages'].apply(lambda x: len(x)if x != {} else 0)
 test['all_languages'] = test['spoken_languages'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_languages:
test['language_' + g] = test['all_languages'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['spoken_languages', 'all_languages'], axis=1)
 test = test.drop(['spoken_languages', 'all_languages'], axis=1 )
train_data['hogar_total'].value_counts()<split>
for i, e in enumerate(train['Keywords'][:5]):
print(i, e )
X_train = train_data.drop('Target', axis=1)
 y_train = train_data['Target'].copy() 
<drop_column>
print('Number of Keywords in films')
 train['Keywords'].apply(lambda x: len(x)if x != {} else 0 ).value_counts().head(10 )
X_train = X_train.drop(['Id', 'idhogar', 'edjefe', 'edjefa'], axis=1 )<feature_engineering>
list_of_keywords = list(train['Keywords'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 plt.figure(figsize =(16, 12))
text = ' '.join(['_'.join(i.split(' ')) for j in list_of_keywords for i in j])
 wordcloud = WordCloud(max_font_size=None, background_color='black', collocations=False,
width=1200, height=1000 ).generate(text)
 plt.imshow(wordcloud)
 plt.title('Top keywords')
 plt.axis("off")
 plt.show()
X_train_da = pd.concat([X_train, X_train, X_train])
 X_train_da['age'] += np.random.randint(2)+ np.random.randint(2)- 2
X_train_da['SQBage'] = X_train_da['age'] ** 2
X_train_da['hogar_total'] += np.random.randint(3)- 1
X_train_da['SQBhogar_total'] = X_train['hogar_total'] ** 2
X_train_da['v2a1'] += np.random.randint(10)* 1000 - 5000
X_train_da = pd.concat([X_train, X_train_da])
 y_train = pd.concat([y_train, y_train, y_train, y_train])
 X_train_da, y_train = shuffle(X_train_da, y_train )<categorify>
train['num_Keywords'] = train['Keywords'].apply(lambda x: len(x)if x != {} else 0)
 train['all_Keywords'] = train['Keywords'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_keywords = [m[0] for m in Counter([i for j in list_of_keywords for i in j] ).most_common(30)]
for g in top_keywords:
train['keyword_' + g] = train['all_Keywords'].apply(lambda x: 1 if g in x else 0)
 test['num_Keywords'] = test['Keywords'].apply(lambda x: len(x)if x != {} else 0)
 test['all_Keywords'] = test['Keywords'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_keywords:
test['keyword_' + g] = test['all_Keywords'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['Keywords', 'all_Keywords'], axis=1)
 test = test.drop(['Keywords', 'all_Keywords'], axis=1 )
X_train_dummy = pd.get_dummies(X_train_da )<normalization>
for i, e in enumerate(train['cast'][:1]):
print(i, e )
imputer = Imputer(strategy='mean')
 imputer.fit(X_train_dummy)
 X_train_dummy = imputer.transform(X_train_dummy)
 scaler = MinMaxScaler()
scaler.fit(X_train_dummy)
 X_train_scaled = scaler.transform(X_train_dummy)
<choose_model_class>
print('Number of casted persons in films')
 train['cast'].apply(lambda x: len(x)if x != {} else 0 ).value_counts().head(10 )
n_classes = 5
y_train_keras = keras.utils.to_categorical(y_train, n_classes)
(n_samples, n_features)= X_train_scaled.shape
model = Sequential()
model.add(Dense(units=500, activation="relu", input_shape=(n_features,)))
 model.add(Dropout(0.2))
model.add(Dense(units=500, activation="relu"))
model.add(Dropout(0.2))
model.add(Dense(units=500, activation="relu"))
model.add(Dropout(0.2))
model.add(Dense(units=100, activation="relu"))
model.add(Dropout(0.2))
model.add(Dense(units=n_classes, activation="softmax"))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 history = model.fit(X_train_scaled, y_train_keras, epochs=100, validation_split=0.1, batch_size=n_samples, verbose=0)
 y_train_pred = model.predict_classes(X_train_scaled, verbose=0)
 f1_score(y_train, y_train_pred, average='macro' )<normalization>
list_of_cast_names = list(train['cast'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_cast_names for i in j] ).most_common(15 )
<compute_train_metric>
list_of_cast_genders = list(train['cast'].apply(lambda x: [i['gender'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_cast_genders for i in j] ).most_common()
<load_from_csv>
list_of_cast_characters = list(train['cast'].apply(lambda x: [i['character'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_cast_characters for i in j] ).most_common(15 )
test_data = pd.read_csv(os.path.join(data_path, 'test.csv'))
tablets = []
for owns, num_tablets in zip(test_data['v18q'], test_data['v18q1']):
if owns == 0:
tablets += [0]
else:
tablets += [num_tablets]
test_data['v18q1'] = tablets
tmp_educ = []
sq_tmp_educ = []
for efe, efa, meduc, sq in zip(test_data['edjefe'], test_data['edjefa'], test_data['meaneduc'], test_data['SQBmeaned']):
new_educ = meduc
if new_educ != new_educ:
if efa == "no":
if efe == "no":
new_educ = 0.0
else:
new_educ = float(efe)
 else:
if efe == "no":
new_educ = float(efa)
 else:
new_educ = float(efe)+ float(efa)
 tmp_educ += [new_educ]
sq_tmp_educ += [new_educ ** 2]
test_data['meaneduc'] = tmp_educ
test_data['SQBmeaned'] = sq_tmp_educ
v2a1 = []
rez_esc = []
for rentpay, rez in zip(test_data['v2a1'], test_data['rez_esc']):
if rentpay != rentpay:
v2a1 += [0]
else:
v2a1 += [rentpay]
if rez != rez:
rez_esc += [0]
else:
rez_esc += [rez]
test_data['rez_esc'] = rez_esc
depend = []
for dependency, children, olds, total in zip(test_data['dependency'], test_data['hogar_nin'], test_data['hogar_mayor'], test_data['hogar_total']):
calc_depend = False
if depend != depend:
calc_depend = True
elif(dependency == "yes" or dependency == "no"):
calc_depend = True
if calc_depend:
i =(children + olds)/(total - children - olds)
 else:
i = float(dependency)
 depend += [i]
test_data['dependency'] = depend
chw = []
for nin, adul in zip(test_data['hogar_nin'], test_data['hogar_adul']):
if adul == 0:
chw += [nin * 2]
else:
chw += [nin/adul]
test_data['child_weight'] =(test_data['hogar_nin'] + test_data['hogar_mayor'])/ test_data['hogar_total']
test_data['child_weight2'] = chw
test_data['child_weight3'] = test_data['r4t1'] / test_data['r4t3']
test_data['work_power'] = test_data['dependency'] * test_data['hogar_adul']
test_data['SQBworker'] = test_data['hogar_adul'] ** 2
test_data['rooms_per_person'] = test_data['rooms'] /(test_data['tamviv'])
 test_data['bedrooms_per_room'] = test_data['bedrooms'] / test_data['rooms']
test_data['female_weight'] = test_data['r4m3'] / test_data['r4t3']
train_hh = test_data.query('parentesco1 == 1')
 v2a1_test_tmp = train_hh.query('v2a1 != v2a1' ).drop(v2a1_drop, axis=1)
 v2a1_test = v2a1_test_tmp.drop('idhogar', axis=1)
(num_target, num_col)= v2a1_test.shape
if(num_target > 0):
v2a1_pred = pd.DataFrame({'ID': v2a1_test_tmp['idhogar'], 'v2a1_pred': forest.predict(v2a1_test)})
 v2a1 = []
for rent, idhh in zip(test_data['v2a1'], test_data['idhogar']):
if rent != rent:
i = 0
for rent_p, index in zip(v2a1_pred['v2a1_pred'], v2a1_pred['ID']):
if index == idhh:
i = rent_p
break
if i == 0:
for rent_org, index in zip(v2a1_train_target, v2a1_train_tmp['idhogar']):
if index == idhh:
i = rent_org
if i < 0:
i = 0
v2a1 += [i]
else:
v2a1 += [rent]
test_data['v2a1'] = v2a1
test_data_drop = test_data.drop(['Id', 'idhogar', 'edjefe', 'edjefa'], axis=1)
 test_data_dummy = pd.get_dummies(test_data_drop)
 test_data_dummy = imputer.transform(test_data_dummy)
 test_data_scaled = scaler.transform(test_data_dummy)
<predict_on_test>
train['num_cast'] = train['cast'].apply(lambda x: len(x)if x != {} else 0)
 top_cast_names = [m[0] for m in Counter([i for j in list_of_cast_names for i in j] ).most_common(15)]
for g in top_cast_names:
train['cast_name_' + g] = train['cast'].apply(lambda x: 1 if g in str(x)else 0)
 train['genders_0_cast'] = train['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
train['genders_1_cast'] = train['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
train['genders_2_cast'] = train['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
top_cast_characters = [m[0] for m in Counter([i for j in list_of_cast_characters for i in j] ).most_common(15)]
for g in top_cast_characters:
train['cast_character_' + g] = train['cast'].apply(lambda x: 1 if g in str(x)else 0)
 test['num_cast'] = test['cast'].apply(lambda x: len(x)if x != {} else 0)
 for g in top_cast_names:
test['cast_name_' + g] = test['cast'].apply(lambda x: 1 if g in str(x)else 0)
 test['genders_0_cast'] = test['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
test['genders_1_cast'] = test['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
test['genders_2_cast'] = test['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
for g in top_cast_characters:
test['cast_character_' + g] = test['cast'].apply(lambda x: 1 if g in str(x)else 0)
 train = train.drop(['cast'], axis=1)
 test = test.drop(['cast'], axis=1 )
y_pred = model.predict_classes(test_data_scaled, verbose=0)
<save_to_csv>
print('Number of casted persons in films')
 train['crew'].apply(lambda x: len(x)if x != {} else 0 ).value_counts().head(10 )
result = pd.DataFrame({'Id':test_data['Id'], 'Target':y_pred})
 result.to_csv('result1.csv', index=False )<set_options>
list_of_crew_names = list(train['crew'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_crew_names for i in j] ).most_common(15 )
%matplotlib inline
sns.set(style = 'darkgrid')
 print(os.listdir(".. /input"))
rcParams['figure.figsize'] = 25, 12.5
rcParams['font.size'] = 50
pd.set_option('display.max_columns',None)
 warnings.filterwarnings('ignore')
 print(os.listdir(".. /input")) 
<load_from_csv>
list_of_crew_jobs = list(train['crew'].apply(lambda x: [i['job'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_crew_jobs for i in j] ).most_common(15 )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.head()<load_from_csv>
list_of_crew_genders = list(train['crew'].apply(lambda x: [i['gender'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_crew_genders for i in j] ).most_common(15 )
submission_sample = pd.read_csv('.. /input/sample_submission.csv' )<define_variables>
list_of_crew_departments = list(train['crew'].apply(lambda x: [i['department'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_crew_departments for i in j] ).most_common(14 )
color_dict = OrderedDict({1: 'red', 2: 'orange', 3: 'blue', 4: 'green'})
 poverty_dict = OrderedDict({1: 'extreme', 2: 'moderate', 3: 'vulnerable', 4: 'non vulnerable'} )<data_type_conversions>
list_of_crew_names = train['crew'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values
Counter([i for j in list_of_crew_names for i in j] ).most_common(15 )
train.loc[:,['dependency','edjefe','edjefa']] =train[['dependency','edjefe','edjefa']].replace(to_replace={'yes':1,'no':0} ).astype(np.float64)
 test.loc[:,['dependency','edjefe','edjefa']] = test[['dependency','edjefe','edjefa']].replace(to_replace={'yes':1,'no':0} ).astype(np.float64 )<groupby>
train['num_crew'] = train['crew'].apply(lambda x: len(x)if x != {} else 0)
 top_crew_names = [m[0] for m in Counter([i for j in list_of_crew_names for i in j] ).most_common(15)]
for g in top_crew_names:
train['crew_name_' + g] = train['crew'].apply(lambda x: 1 if g in str(x)else 0)
 train['genders_0_crew'] = train['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
train['genders_1_crew'] = train['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
train['genders_2_crew'] = train['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
top_crew_names = [m[0] for m in Counter([i for j in list_of_crew_names for i in j] ).most_common(15)]
top_crew_jobs = [m[0] for m in Counter([i for j in list_of_crew_jobs for i in j] ).most_common(15)]
for j in top_crew_jobs:
train['jobs_' + j] = train['crew'].apply(lambda x: sum([1 for i in x if i['job'] == j]))
top_crew_departments = [m[0] for m in Counter([i for j in list_of_crew_departments for i in j] ).most_common(15)]
for j in top_crew_departments:
train['departments_' + j] = train['crew'].apply(lambda x: sum([1 for i in x if i['department'] == j]))
test['num_crew'] = test['crew'].apply(lambda x: len(x)if x != {} else 0)
 for g in top_crew_names:
test['crew_name_' + g] = test['crew'].apply(lambda x: 1 if g in str(x)else 0)
 test['genders_0_crew'] = test['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
test['genders_1_crew'] = test['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
test['genders_2_crew'] = test['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
for j in top_crew_jobs:
test['jobs_' + j] = test['crew'].apply(lambda x: sum([1 for i in x if i['job'] == j]))
for j in top_crew_departments:
test['departments_' + j] = test['crew'].apply(lambda x: sum([1 for i in x if i['department'] == j]))
train = train.drop(['crew'], axis=1)
 test = test.drop(['crew'], axis=1 )
household_checktarget =train.groupby('idhogar')['Target'].apply(lambda x: x.nunique() == 1)
 idhogar_mult_target = household_checktarget[household_checktarget!=True].index
print('Number of households where the poverty level is more than one:{}'.format(len(idhogar_mult_target)) )<feature_engineering>
train['log_revenue'] = np.log1p(train['revenue'] )
for idh in idhogar_mult_target:
actual_povertylevel = train.loc[(( train.parentesco1==1)&(train.idhogar==idh)) ,'Target']
train.loc[(train.idhogar==idh),'Target'] = actual_povertylevel<groupby>
train['log_budget'] = np.log1p(train['budget'])
 test['log_budget'] = np.log1p(test['budget'] )
household_num_head = train.groupby('idhogar')['parentesco1'].sum()
idhogar_nohead = household_num_head[household_num_head!=1].index
print('Number of households where there are no head:{}'.format(len(idhogar_nohead)) )<filter>
train['has_homepage'] = 0
train.loc[train['homepage'].isnull() == False, 'has_homepage'] = 1
test['has_homepage'] = 0
test.loc[test['homepage'].isnull() == False, 'has_homepage'] = 1
train[train.idhogar == 'f2bfa75c4']<define_variables>
vectorizer = TfidfVectorizer(
 sublinear_tf=True,
analyzer='word',
token_pattern=r'\w{1,}',
ngram_range=(1, 2),
min_df=5)
 overview_text = vectorizer.fit_transform(train['overview'].fillna(''))
linreg = LinearRegression()
linreg.fit(overview_text, train['log_revenue'])
 eli5.show_weights(linreg, vec=vectorizer, top=20, feature_filter=lambda x: x != '<BIAS>' )
id_columns = ['Id', 'idhogar', 'Target']
household_boolean = ['hacdor', 'hacapo', 'v14a', 'refrig', 'paredblolad', 'paredzocalo',
'paredpreb','pisocemento', 'pareddes', 'paredmad',
'paredzinc', 'paredfibras', 'paredother', 'pisomoscer', 'pisoother',
'pisonatur', 'pisonotiene', 'pisomadera',
'techozinc', 'techoentrepiso', 'techocane', 'techootro', 'cielorazo',
'abastaguadentro', 'abastaguafuera', 'abastaguano',
'public', 'planpri', 'noelec', 'coopele', 'sanitario1',
'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6',
'energcocinar1', 'energcocinar2', 'energcocinar3', 'energcocinar4',
'elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4',
'elimbasu5', 'elimbasu6', 'epared1', 'epared2', 'epared3',
'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3',
'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5',
'computer', 'television', 'lugar1', 'lugar2', 'lugar3',
'lugar4', 'lugar5', 'lugar6', 'area1', 'area2']
household_continuos = ['v2a1', 'dependency', 'edjefe', 'edjefa', 'meaneduc', 'overcrowding']
household_ordered = [ 'rooms', 'r4h1', 'r4h2', 'r4h3', 'r4m1','r4m2','r4m3', 'r4t1', 'r4t2',
'r4t3', 'v18q1', 'tamhog','tamviv','hhsize','hogar_nin',
'hogar_adul','hogar_mayor','hogar_total', 'bedrooms', 'qmobilephone']
ind_bool = ['v18q', 'dis', 'male', 'female', 'estadocivil1', 'estadocivil2', 'estadocivil3',
'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco1', 'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5',
'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12', 'instlevel1', 'instlevel2', 'instlevel3',
'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8',
'instlevel9', 'mobilephone']
ind_ordered = ['rez_esc', 'escolari', 'age']
squared_common = ['SQBescolari','SQBage', 'SQBhogar_total','SQBedjefe','SQBhogar_nin','SQBovercrowding',
'SQBdependency', 'SQBmeaned']<groupby>
test.loc[test['release_date'].isnull() == True, 'release_date'] = '01/01/98'
feature_train_ind_cont = train.groupby('idhogar')[ind_ordered].agg(['mean', 'max', 'min', 'sum'])
 feature_test_ind_cont = test.groupby('idhogar')[ind_ordered].agg(['mean', 'max', 'min', 'sum'])
 new_cols = []
for col in feature_train_ind_cont.columns.levels[0]:
for stat in feature_train_ind_cont.columns.levels[1]:
new_cols.append(f'{col}-{stat}')
 feature_train_ind_cont.columns = new_cols
feature_test_ind_cont.columns = new_cols
feature_train_ind_cont.reset_index(inplace=True)
 feature_test_ind_cont.reset_index(inplace=True )<count_missing_values>
def fix_date(x):

 year = x.split('/')[2]
if int(year)<= 19:
return x[:-2] + '20' + year
else:
return x[:-2] + '19' + year
feature_train_ind_cont['rez_esc-mean'].isnull().sum() /len(feature_train_ind_cont )<merge>
train['release_date'] = train['release_date'].apply(lambda x: fix_date(x))
test['release_date'] = test['release_date'].apply(lambda x: fix_date(x))
train['release_date'] = pd.to_datetime(train['release_date'])
 test['release_date'] = pd.to_datetime(test['release_date'] )
feature_train_ind = pd.merge(feature_train_ind_bool,feature_train_ind_cont,on='idhogar',how='inner')
 feature_test_ind = pd.merge(feature_test_ind_bool,feature_test_ind_cont,on='idhogar',how='inner')
 print('Individual training features data shape:{}
Individual test features data shape:{}'.format(feature_train_ind.shape,feature_test_ind.shape))<merge>
def process_date(df):
date_parts = ["year", "weekday", "month", 'weekofyear', 'day', 'quarter']
for part in date_parts:
part_col = 'release_date' + "_" + part
df[part_col] = getattr(df['release_date'].dt, part ).astype(int)
 return df
train = process_date(train)
 test = process_date(test )
feature_train = pd.merge(feature_train_household,feature_train_ind,on='idhogar',how='left')
 feature_test = pd.merge(feature_test_household,feature_test_ind,on='idhogar',how='left')
 print('Combined features train data shape:{}
Combined features test data shape:{}'.format(feature_train.shape,feature_test.shape))<split>
train = train.drop(['homepage', 'imdb_id', 'poster_path', 'release_date', 'status', 'log_revenue'], axis=1)
 test = test.drop(['homepage', 'imdb_id', 'poster_path', 'release_date', 'status'], axis=1 )
target_train = feature_train['Target']-1
test_idhogar = feature_test['idhogar'].values
try:
feature_train.drop('Target',axis=1,inplace=True)
 feature_train.drop(['Id','idhogar'],axis=1,inplace=True)
 feature_test.drop(['Id','idhogar'],axis=1,inplace=True)
 except:
print("Couldnot drop all columns")
 feature_train, feature_test = feature_train.align(feature_test, axis = 1, join = 'inner')
 feature_X_train,feature_X_valid,target_y_train,target_y_valid = train_test_split(feature_train,target_train,test_size=0.2,stratify=target_train,random_state=123)
<compute_test_metric>
for col in train.columns:
if train[col].nunique() == 1:
print(col)
 train = train.drop([col], axis=1)
 test = test.drop([col], axis=1 )
def macro_f1_score(labels, predictions):
predictions = predictions.reshape(len(np.unique(labels)) , -1 ).argmax(axis = 0)
 metric_value = f1_score(labels, predictions, average = 'macro')
 return 'macro_f1', metric_value, True<choose_model_class>
for col in ['original_language', 'collection_name', 'all_genres']:
le = LabelEncoder()
le.fit(list(train[col].fillna('')) + list(test[col].fillna('')))
 train[col] = le.transform(train[col].fillna('' ).astype(str))
test[col] = le.transform(test[col].fillna('' ).astype(str))
nfolds=5
sfold = StratifiedKFold(n_splits= nfolds,shuffle=True )<define_search_space>
train_texts = train[['title', 'tagline', 'overview', 'original_title']]
test_texts = test[['title', 'tagline', 'overview', 'original_title']]
<init_hyperparams>
for col in ['title', 'tagline', 'overview', 'original_title']:
train['len_' + col] = train[col].fillna('' ).apply(lambda x: len(str(x)))
 train['words_' + col] = train[col].fillna('' ).apply(lambda x: len(str(x.split(' '))))
train = train.drop(col, axis=1)
 test['len_' + col] = test[col].fillna('' ).apply(lambda x: len(str(x)))
 test['words_' + col] = test[col].fillna('' ).apply(lambda x: len(str(x.split(' '))))
test = test.drop(col, axis=1 )
dict_optimum_params = {'colsample_bytree': 0.605756972263847,
'learning_rate': 0.01,
'min_child_samples': 19,
'num_leaves': 29,
'subsample': 0.5522796688057564}<compute_test_metric>
train.loc[train['id'] == 16,'revenue'] = 192864
train.loc[train['id'] == 90,'budget'] = 30000000
train.loc[train['id'] == 118,'budget'] = 60000000
train.loc[train['id'] == 149,'budget'] = 18000000
train.loc[train['id'] == 313,'revenue'] = 12000000
train.loc[train['id'] == 451,'revenue'] = 12000000
train.loc[train['id'] == 464,'budget'] = 20000000
train.loc[train['id'] == 470,'budget'] = 13000000
train.loc[train['id'] == 513,'budget'] = 930000
train.loc[train['id'] == 797,'budget'] = 8000000
train.loc[train['id'] == 819,'budget'] = 90000000
train.loc[train['id'] == 850,'budget'] = 90000000
train.loc[train['id'] == 1112,'budget'] = 7500000
train.loc[train['id'] == 1131,'budget'] = 4300000
train.loc[train['id'] == 1359,'budget'] = 10000000
train.loc[train['id'] == 1542,'budget'] = 1
train.loc[train['id'] == 1570,'budget'] = 15800000
train.loc[train['id'] == 1571,'budget'] = 4000000
train.loc[train['id'] == 1714,'budget'] = 46000000
train.loc[train['id'] == 1721,'budget'] = 17500000
train.loc[train['id'] == 1865,'revenue'] = 25000000
train.loc[train['id'] == 2268,'budget'] = 17500000
train.loc[train['id'] == 2491,'revenue'] = 6800000
train.loc[train['id'] == 2602,'budget'] = 31000000
train.loc[train['id'] == 2612,'budget'] = 15000000
train.loc[train['id'] == 2696,'budget'] = 10000000
train.loc[train['id'] == 2801,'budget'] = 10000000
test.loc[test['id'] == 3889,'budget'] = 15000000
test.loc[test['id'] == 6733,'budget'] = 5000000
test.loc[test['id'] == 3197,'budget'] = 8000000
test.loc[test['id'] == 6683,'budget'] = 50000000
test.loc[test['id'] == 5704,'budget'] = 4300000
test.loc[test['id'] == 6109,'budget'] = 281756
test.loc[test['id'] == 7242,'budget'] = 10000000
test.loc[test['id'] == 7021,'budget'] = 17540562
test.loc[test['id'] == 5591,'budget'] = 4000000
test.loc[test['id'] == 4282,'budget'] = 20000000
power_six = train.id[train.budget > 1000][train.revenue < 100]
for k in power_six :
train.loc[train['id'] == k,'revenue'] = train.loc[train['id'] == k,'revenue'] * 1000000
test_predictions_df=train_eval(feature_train,feature_test,target_train )<compute_test_metric>
def new_features(df):
df['budget_to_popularity'] = df['budget'] / df['popularity']
df['budget_to_runtime'] = df['budget'] / df['runtime']
df['_budget_year_ratio'] = df['budget'] /(df['release_date_year'] * df['release_date_year'])
 df['_releaseYear_popularity_ratio'] = df['release_date_year'] / df['popularity']
df['_releaseYear_popularity_ratio2'] = df['popularity'] / df['release_date_year']
df['runtime_to_mean_year'] = df['runtime'] / df.groupby("release_date_year")["runtime"].transform('mean')
 df['popularity_to_mean_year'] = df['popularity'] / df.groupby("release_date_year")["popularity"].transform('mean')
 df['budget_to_mean_year'] = df['budget'] / df.groupby("release_date_year")["budget"].transform('mean')
 return df
<filter>
test = new_features(test)
 train = new_features(train )
submission_df = test.loc[:,['Id','idhogar']]<merge>
test['runtime'] = test['runtime'].fillna(( test['runtime'].mean()))
train['runtime'] = train['runtime'].fillna(( test['runtime'].mean()))
all_data = pd.concat(( train, test)).reset_index(drop=True)
 all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 missing_data.head(20 )
submission = pd.merge(submission_df,test_predictions_df[['idhogar','Target']],on='idhogar',how='left' ).drop('idhogar',axis=1 )<data_type_conversions>
skewness_test = skewness_test[abs(skewness_test)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness_test.shape[0]))
skewed_features_test = skewness_test.index
lam_test = 0.15
for feat_test in skewed_features_test:
if feat_test !='revenue':
test[feat_test] = boxcox1p(test[feat_test], lam_test )
submission['Target'] = submission['Target'].fillna(4 ).astype(np.int8 )<save_to_csv>
skewness_train = skewness_train[abs(skewness_train)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness_train.shape[0]))
skewed_features_train = skewness_train.index
lam_train = 0.15
for feat_train in skewed_features_train:
if feat_train !='revenue':
train[feat_train] = boxcox1p(train[feat_train], lam_train )
submission.to_csv('sample_submission.csv',index=False )<set_options>
X = train.drop(['id', 'revenue'], axis=1)
 y = np.log1p(train['revenue'])
 X_testfile = test.drop(['id'], axis=1 )
%matplotlib inline
warnings.filterwarnings("ignore" )<categorify>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3 )
def encode_data(df):
df['idhogar'] = LabelEncoder().fit_transform(df['idhogar'])
 def feature_importance(forest, X_train):
ranked_list = []
importances = forest.feature_importances_
indices = np.argsort(importances)[::-1]
print("Feature ranking:")
 for f in range(X_train.shape[1]):
print("%d.feature %d(%f)" %(f + 1, indices[f], importances[indices[f]])+ " - " + X_train.columns[indices[f]])
 ranked_list.append(X_train.columns[indices[f]])
 return ranked_list<feature_engineering>
params = {'num_leaves': 30,
'min_data_in_leaf': 20,
'objective': 'regression',
'max_depth': 5,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.2,
"verbosity": -1}
model_lgb = lgb.LGBMRegressor(**params, n_estimators = 20000, nthread = 4, n_jobs = -1)
 model_lgb.fit(X_train, y_train
,eval_set=[(X_train, y_train),(X_test, y_test)], eval_metric='rmse',
verbose=1000, early_stopping_rounds=200 )
def do_features(df):
feats_div = [('children_fraction', 'r4t1', 'r4t3'), 
('working_man_fraction', 'r4h2', 'r4t3'), 
('all_man_fraction', 'r4h3', 'r4t3'), 
('human_density', 'tamviv', 'rooms'), 
('human_bed_density', 'tamviv', 'bedrooms'), 
('rent_per_person', 'v2a1', 'r4t3'), 
('rent_per_room', 'v2a1', 'rooms'), 
('mobile_density', 'qmobilephone', 'r4t3'), 
('tablet_density', 'v18q1', 'r4t3'), 
('mobile_adult_density', 'qmobilephone', 'r4t2'), 
('tablet_adult_density', 'v18q1', 'r4t2'),
]
feats_sub = [('people_not_living', 'tamhog', 'tamviv'), 
('people_weird_stat', 'tamhog', 'r4t3')]
for f_new, f1, f2 in feats_div:
df['fe_' + f_new] =(df[f1] / df[f2] ).astype(np.float32)
 for f_new, f1, f2 in feats_sub:
df['fe_' + f_new] =(df[f1] - df[f2] ).astype(np.float32)
 aggs_num = {'age': ['min', 'max', 'mean'],
'escolari': ['min', 'max', 'mean']
}
aggs_cat = {'dis': ['mean']}
for s_ in ['estadocivil', 'parentesco', 'instlevel']:
for f_ in [f_ for f_ in df.columns if f_.startswith(s_)]:
aggs_cat[f_] = ['mean', 'count']
for name_, df_ in [('18', df.query('age >= 18')) ]:
df_agg = df_.groupby('idhogar' ).agg({**aggs_num, **aggs_cat} ).astype(np.float32)
 df_agg.columns = pd.Index(['agg' + name_ + '_' + e[0] + "_" + e[1].upper() for e in df_agg.columns.tolist() ])
 df = df.join(df_agg, how='left', on='idhogar')
 del df_agg
df.drop(['Id'], axis=1, inplace=True)
 return df<categorify>
eli5.show_weights(model_lgb, feature_filter=lambda x: x != '<BIAS>' )
def convert_OHE2LE(df):
tmp_df = df.copy(deep=True)
 for s_ in ['pared', 'piso', 'techo', 'abastagua', 'sanitario', 'energcocinar', 'elimbasu',
'epared', 'etecho', 'eviv', 'estadocivil', 'parentesco',
'instlevel', 'lugar', 'tipovivi',
'manual_elec']:
if 'manual_' not in s_:
cols_s_ = [f_ for f_ in df.columns if f_.startswith(s_)]
elif 'elec' in s_:
cols_s_ = ['public', 'planpri', 'noelec', 'coopele']
sum_ohe = tmp_df[cols_s_].sum(axis=1 ).unique()
if 0 in sum_ohe:
print('The OHE in {} is incomplete.A new column will be added before label encoding' 
.format(s_))
col_dummy = s_+'_dummy'
tmp_df[col_dummy] =(tmp_df[cols_s_].sum(axis=1)== 0 ).astype(np.int8)
 cols_s_.append(col_dummy)
 sum_ohe = tmp_df[cols_s_].sum(axis=1 ).unique()
if 0 in sum_ohe:
print("The category completion did not work")
 tmp_cat = tmp_df[cols_s_].idxmax(axis=1)
 tmp_df[s_ + '_LE'] = LabelEncoder().fit_transform(tmp_cat ).astype(np.int16)
 if 'parentesco1' in cols_s_:
cols_s_.remove('parentesco1')
 tmp_df.drop(cols_s_, axis=1, inplace=True)
 return tmp_df<load_from_csv>
prediction_lgb_train=model_lgb.predict(X_train )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 test_ids = test.Id<categorify>
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
print(rmsle(y_train, prediction_lgb_train))
def process_df(df_):
encode_data(df_)
 return do_features(df_)
 train = process_df(train)
 test = process_df(test )<feature_engineering>
prediction_lgb_test=model_lgb.predict(X_test )
train['dependency'] = np.sqrt(train['SQBdependency'])
 test['dependency'] = np.sqrt(test['SQBdependency'])
 train.loc[train['edjefa'] == "no", "edjefa"] = 0
train.loc[train['edjefe'] == "no", "edjefe"] = 0
test.loc[test['edjefa'] == "no", "edjefa"] = 0
test.loc[test['edjefe'] == "no", "edjefe"] = 0
train.loc[(train['edjefa'] == "yes")&(train['parentesco1'] == 1), "edjefa"] = train.loc[(train['edjefa'] == "yes")&(train['parentesco1'] == 1), "escolari"]
train.loc[(train['edjefe'] == "yes")&(train['parentesco1'] == 1), "edjefe"] = train.loc[(train['edjefe'] == "yes")&(train['parentesco1'] == 1), "escolari"]
test.loc[(test['edjefa'] == "yes")&(test['parentesco1'] == 1), "edjefa"] = test.loc[(test['edjefa'] == "yes")&(test['parentesco1'] == 1), "escolari"]
test.loc[(test['edjefe'] == "yes")&(test['parentesco1'] == 1), "edjefe"] = test.loc[(test['edjefe'] == "yes")&(test['parentesco1'] == 1), "escolari"]
train.loc[train['edjefa'] == "yes", "edjefa"] = 4
train.loc[train['edjefe'] == "yes", "edjefe"] = 4
test.loc[test['edjefa'] == "yes", "edjefa"] = 4
test.loc[test['edjefe'] == "yes", "edjefe"] = 4
train['edjefe'] = train['edjefe'].astype("int")
 train['edjefa'] = train['edjefa'].astype("int")
 test['edjefe'] = test['edjefe'].astype("int")
 test['edjefa'] = test['edjefa'].astype("int")
 train['edjef'] = np.max(train[['edjefa','edjefe']], axis=1)
 test['edjef'] = np.max(test[['edjefa','edjefe']], axis=1)
 train['v2a1']=train['v2a1'].fillna(0)
 test['v2a1']=test['v2a1'].fillna(0)
 test['v18q1']=test['v18q1'].fillna(0)
 train['v18q1']=train['v18q1'].fillna(0)
 train['rez_esc']=train['rez_esc'].fillna(0)
 test['rez_esc']=test['rez_esc'].fillna(0)
 train.loc[train.meaneduc.isnull() , "meaneduc"] = 0
train.loc[train.SQBmeaned.isnull() , "SQBmeaned"] = 0
test.loc[test.meaneduc.isnull() , "meaneduc"] = 0
test.loc[test.SQBmeaned.isnull() , "SQBmeaned"] = 0
train.loc[(train.v14a == 1)&(train.sanitario1 == 1)&(train.abastaguano == 0), "v14a"] = 0
train.loc[(train.v14a == 1)&(train.sanitario1 == 1)&(train.abastaguano == 0), "sanitario1"] = 0
test.loc[(test.v14a == 1)&(test.sanitario1 == 1)&(test.abastaguano == 0), "v14a"] = 0
test.loc[(test.v14a == 1)&(test.sanitario1 == 1)&(test.abastaguano == 0), "sanitario1"] = 0<concatenate>
print(rmsle(y_test, prediction_lgb_test))
def train_test_apply_func(train_, test_, func_):
test_['Target'] = 0
xx = pd.concat([train_, test_])
 xx_func = func_(xx)
 train_ = xx_func.iloc[:train_.shape[0], :]
test_ = xx_func.iloc[train_.shape[0]:, :].drop('Target', axis=1)
 del xx, xx_func
return train_, test_<categorify>
prediction_lgb_testfile=model_lgb.predict(X_testfile )
<categorify><EOS> 
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['revenue'] = np.expm1(prediction_lgb_testfile)
 sub.to_csv("lgb.csv", index=False )