Kaggle competition: TMDB Box Office Prediction
<SOS> metric: RMSLE Kaggle data source: tmdb-box-office-prediction<predict_on_test>
assert sys.version_info >=(3, 5)
 assert sklearn.__version__ >= "0.20"
%matplotlib inline
warnings.filterwarnings(action="ignore")
 pd.set_option('display.max_columns', 500)
 pd.set_option('display.max_rows', 500)
 print(os.listdir(".. /input"))
def submit(model, train, train_labels, test, test_ids):
model.fit(train, train_labels)
 predictions = model.predict(test)
 predictions = pd.DataFrame({'idhogar' : test_ids,
'Target' : predictions})
 submission = submission_base.merge(predictions,
on = 'idhogar',
how='left' ).drop(columns = ['idhogar'])
 submission['Target'] = submission['Target'].fillna(4 ).astype(np.int8)
 return submission<save_to_csv>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
rf_submission = submit(RandomForestClassifier(n_estimators = 100,
random_state=10, n_jobs = -1),
train_set, train_labels, test_set, test_ids)
 rf_submission.to_csv('rf_submission.csv', index=False )<concatenate>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
test_set = pd.DataFrame(test_set, columns=features)
 train_set, test_set = train_set.align(test_set, axis = 1, join ='inner')
 features = list(train_set.columns )<choose_model_class>
dict_columns = ['belongs_to_collection', 'genres', 'production_companies',
'production_countries', 'spoken_languages', 'Keywords', 'cast', 'crew']
def text_to_dict(df):
for column in dict_columns:
df[column] = df[column].apply(lambda x: {} if pd.isna(x)else ast.literal_eval(x))
return df
train = text_to_dict(train)
 test = text_to_dict(test )
estimator = RandomForestClassifier(random_state = 10 , n_estimators = 100, n_jobs= -1)
 selector = RFECV(estimator, step = 1, cv =3, scoring=scorer, n_jobs=-1 )<train_model>
train['source']='train'
test['source']='test'
data = pd.concat([train, test],ignore_index=True)
 print(train.shape, test.shape, data.shape )
selector.fit(train_set, train_labels )<prepare_output>
print("Belongs to a collection or not:
%s" %(data['belongs_to_collection'].apply(lambda x: len(x)if x != {} else 0 ).value_counts()))
rankings = pd.DataFrame({'feature' : list(train_set.columns), 'rank': list(selector.ranking_)} ).sort_values('rank')
 rankings.head(10 )<categorify>
def dict_one_hot_code(dataset,name_column, name_new_column):
list_of_values = list(dataset[name_column].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_values for i in j] ).most_common(20)
 top_list_values = [m[0] for m in Counter([i for j in list_of_values for i in j] ).most_common(20)]
for g in top_list_values:
dataset[name_new_column + "_" + g] = dataset[name_column].apply(lambda x: 1 if g in str(x)else 0)
 def counting_number_of_values_dict(dataset, name_column):
return dataset[name_column].apply(lambda x: len(x)if x!= {} else 0 )
train_selected = selector.transform(train_set)
 test_selected = selector.transform(test_set )<create_dataframe>
data['collection_name'] = data['belongs_to_collection'].apply(lambda x: x[0]['name'] if x != {} else "None")
 data['is_part_of_collection'] = data['belongs_to_collection'].apply(lambda x: len(x)if x != {} else 0)
 data.drop('belongs_to_collection', axis = 1, inplace=True )
selected_features = train_set.columns[np.where(selector.ranking_==1)]
train_selected = pd.DataFrame(train_selected, columns = selected_features)
 test_selected = pd.DataFrame(test_selected, columns = selected_features )<compute_train_metric>
data['genres'].apply(lambda x: x[0]['name'] if x != {} else 0 ).value_counts()
model_results = cv_model(train_selected, train_labels, model, 'RF-SEL', model_results )<compute_test_metric>
data['genre'] = data['genres'].apply(lambda x: x[0]['name'] if x != {} else "None")
 data.drop('genres', axis=1, inplace=True )
def macro_f1_score(labels, predictions):
predictions = predictions.reshape(len(np.unique(labels)) , -1 ).argmax(axis = 0)
 metric_value = f1_score(labels, predictions, average ='macro')
 return 'macro_f1', metric_value, True<choose_model_class>
list_of_cast_names = list(data['cast'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_cast_names for i in j] ).most_common(20 )
def model_gbm(features, labels, test_features, test_ids, nfolds = 5, return_preds = False, hyp = None):

 feature_names = list(features.columns)
 if hyp is not None:
if 'n_estimators' in hyp:
del hyp['n_estimators']
params = hyp
else:
params = {'boosting_type': 'dart',
'colsample_bytree': 0.88,
'learning_rate': 0.028,
'min_child_samples': 10,
'num_leaves': 36, 'reg_alpha': 0.76,
'reg_lambda': 0.43,
'subsample_for_bin': 40000,
'subsample': 0.54,
'class_weight': 'balanced'}
model = lgb.LGBMClassifier(**params, objective = 'multiclass',
n_jobs = -1, n_estimators = 10000,
random_state = 10)
 strkfold = StratifiedKFold(n_splits = nfolds, shuffle = True)
 predictions = pd.DataFrame()
importances = np.zeros(len(feature_names))
features = np.array(features)
 test_features = np.array(test_features)
 labels = np.array(labels ).reshape(( -1))
valid_scores = []
for i,(train_indices, valid_indices)in enumerate(strkfold.split(features, labels)) :
fold_predictions = pd.DataFrame()
X_train = features[train_indices]
X_valid = features[valid_indices]
y_train = labels[train_indices]
y_valid = labels[valid_indices]
model.fit(X_train, y_train, early_stopping_rounds = 100, eval_metric = macro_f1_score,
eval_set = [(X_train, y_train),(X_valid, y_valid)] , eval_names = ['train', 'valid'], verbose = 200)
 valid_scores.append(model.best_score_['valid']['macro_f1'])
 fold_probabilitites = model.predict_proba(test_features)
 for j in range(4):
fold_predictions[(j + 1)] = fold_probabilitites[:, j]
fold_predictions['idhogar'] = test_ids
fold_predictions['fold'] =(i+1)
 predictions = predictions.append(fold_predictions)
 importances += model.feature_importances_ / nfolds
display(f'Fold {i + 1}, Validation Score: {round(valid_scores[i], 5)}, Estimators Trained: {model.best_iteration_}')
 feature_importances = pd.DataFrame({'feature': feature_names,
'importance': importances})
 valid_scores = np.array(valid_scores)
 display(f'{nfolds} cross validation score: {round(valid_scores.mean() , 5)} with std: {round(valid_scores.std() , 5)}.')
 if return_preds:
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 return predictions, feature_importances
predictions = predictions.groupby('idhogar', as_index = False ).mean()
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 predictions = predictions.drop(columns = ['fold'])
 submission = submission_base.merge(predictions[['idhogar', 'Target']], on = 'idhogar', how = 'left' ).drop(columns = ['idhogar'])
 submission['Target'] = submission['Target'].fillna(4 ).astype(np.int8)
 return submission, feature_importances, valid_scores<compute_train_metric>
data['cast'].apply(lambda x: len(x)if x != {} else 0 ).value_counts()
%%capture --no-display
predictions, gbm_fi = model_gbm(train_set , train_labels, test_set, test_ids, return_preds=True )<compute_train_metric>
data['number_cast'] = counting_number_of_values_dict(data,'cast' )
%%capture
submission, gbm_fi, valid_scores = model_gbm(train_set, train_labels, test_set, test_ids, return_preds = False)
<prepare_output>
list_of_cast_characters = list(data['cast'].apply(lambda x: [i['character'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_cast_characters for i in j] ).most_common(15 )
submission = submission[['id', 'Target']]<save_to_csv>
top_cast_names = [m[0] for m in Counter([i for j in list_of_cast_names for i in j] ).most_common(15)]
for g in top_cast_names:
data['cast_name_' + g] = data['cast'].apply(lambda x: 1 if g in str(x)else 0 )
submission.to_csv('gbm_baseline.csv', index=False )<set_options>
top_cast_characters = [m[0] for m in Counter([i for j in list_of_cast_characters for i in j] ).most_common(15)]
for g in top_cast_characters:
data['cast_character_' + g] = data['cast'].apply(lambda x: 1 if g in str(x)else 0 )
pd.options.display.max_columns = 150 
<load_from_csv>
data.drop('cast', axis=1, inplace=True )
train = pd.read_csv('.. /input/train.csv')
 train.info()
train.head()<load_from_csv>
counting_number_of_values_dict(data,'crew' ).value_counts()
test = pd.read_csv('.. /input/test.csv')
 test.info()
test.head()<create_dataframe>
data['crew_total'] = counting_number_of_values_dict(data, 'crew' )
train_na= pd.DataFrame(( train.isnull().sum().values),index=train.columns, columns=['isNA'] ).sort_values(by=['isNA'],ascending=False)
 if train_na.loc[train_na.loc[:,'isNA']>0,:].shape[0]>1 :
print("Train Dataset")
 print(train_na.loc[train_na.loc[:,'isNA']> 0,])
 else:
print('no NA in train set')
 test_na= pd.DataFrame(( test.isnull().sum().values),index=test.columns, columns=['isNA'] ).sort_values(by=['isNA'],ascending=False)
 if train_na.loc[train_na.loc[:,'isNA']>0,:].shape[0]>1 :
print("")
 print("Test Dataset")
 print(test_na.loc[test_na.loc[:,'isNA']> 0,])
 else:
print('no NA in test set' )<feature_engineering>
dict_one_hot_code(data,name_column='crew',name_new_column='crew_name' )
train.loc[:,"meaneduc"].mode()
train.loc[train.loc[:,"meaneduc"].isnull() ==True,"meaneduc"] = 6
train.loc[train.loc[:,"SQBmeaned"].isnull() ==True,"SQBmeaned"] = 36
test.loc[:,"meaneduc"].mode()
test.loc[test.loc[:,"meaneduc"].isnull() ==True,"meaneduc"] = 6
test.loc[test.loc[:,"SQBmeaned"].isnull() ==True,"SQBmeaned"] = 36
train.loc[train.loc[:,"rez_esc"].isnull() ==True,"rez_esc"] = 0
train.loc[train.loc[:,"v18q1"].isnull() ==True,"v18q1"] = 0
train.loc[train.loc[:,"v2a1"].isnull() ==True,"v2a1"] = 0
test.loc[test.loc[:,"rez_esc"].isnull() ==True,"rez_esc"] = 0
test.loc[test.loc[:,"v18q1"].isnull() ==True,"v18q1"] = 0
test.loc[test.loc[:,"v2a1"].isnull() ==True,"v2a1"] = 0 
<create_dataframe>
data.drop('crew', axis=1, inplace=True )
train_na= pd.DataFrame(( train.isnull().sum().values),index=train.columns, columns=['isNA'] ).sort_values(by=['isNA'],ascending=False)
 if train_na.loc[train_na.loc[:,'isNA']>0,:].shape[0]>1 :
train_na.loc[train_na.loc[:,'isNA']> 0,]
else:
print('No NA in train set')
 test_na= pd.DataFrame(( test.isnull().sum().values),index=test.columns, columns=['isNA'] ).sort_values(by=['isNA'],ascending=False)
 if train_na.loc[train_na.loc[:,'isNA']>0,:].shape[0]>1 :
test_na.loc[test_na.loc[:,'isNA']> 0,]
else:
print('No NA in test set' )<count_unique_values>
counting_number_of_values_dict(data,'Keywords' ).value_counts()
target_Discrepancy=(train.groupby('idhogar')['Target'].nunique() >1)
 num_unique_households = train["idhogar"].unique().shape[0]
print('There are',target_Discrepancy.sum() ,'households with contradicting targets, out of', num_unique_households, 'households in the train dataset.' )<feature_engineering>
data['keywords_num'] = counting_number_of_values_dict(data,'Keywords' )
for HH in HHID_Discrepancy:
Targets=(train.loc[train['idhogar']==HH,'Target'])
 if Targets.mode().shape[0] >1:
for i in Targets.index:
if train.loc[i,'parentesco1']==1:
HeadTarget= train.loc[i,"Target"]
for i in Targets.index:
train.loc[i,'Target']=HeadTarget
elif Targets.mode().shape[0]==1:
for i in Targets.index:
TrueTarget=int(Targets.mode())
 train.loc[i,'Target']=TrueTarget 
<count_unique_values>
list_of_keywords = list(data['Keywords'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values )
target_Discrepancy=(train.groupby('idhogar')['Target'].nunique() >1)
 print('There are ',target_Discrepancy.sum() ,'households with contradicting targets, out of 2988 households in the train dataset')
 train.head()
train.shape<drop_column>
data['all_Keywords'] = data['Keywords'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '' )
train=train.drop(columns=train.columns[133:142],axis=1)
 test=test.drop(columns=test.columns[133:142],axis=1 )<define_variables>
top_keywords = [m[0] for m in Counter([i for j in list_of_keywords for i in j] ).most_common(30)]
for g in top_keywords:
data['keyword_' + g] = data['all_Keywords'].apply(lambda x: 1 if g in x else 0 )
numColumnsHH= ["v2a1","rooms","v18q1","qmobilephone","r4h3","r4m3","r4t3","tamhog","hhsize","hogar_nin","hogar_adul",
"hogar_mayor","hogar_total","meaneduc","bedrooms","overcrowding"]<define_variables>
data.drop(['Keywords','all_Keywords'], axis=1, inplace=True )
wall = ["paredblolad","paredzocalo","paredpreb","pareddes","paredmad","paredzinc","paredfibras","paredother"]
floor = ["pisomoscer","pisocemento","pisoother","pisonatur","pisonotiene","pisomadera"]
roof = ['techozinc','techoentrepiso','techocane','techootro', 'cielorazo']
water = ["abastaguadentro","abastaguafuera","abastaguano"]
electric = ["public","planpri","noelec","coopele"]
toilet = ['sanitario1','sanitario2','sanitario3','sanitario5','sanitario6']
energy = ['energcocinar1','energcocinar2','energcocinar3','energcocinar4']
rubbish = ['elimbasu1','elimbasu2','elimbasu3','elimbasu4','elimbasu5','elimbasu6']
wall_cond = ['epared1','epared2','epared3']
roof_cond = ['etecho1','etecho2','etecho3']
floor_cond = ['eviv1','eviv2','eviv3']
col_types = [wall, floor, roof, water, electric, toilet, energy, rubbish, wall_cond, roof_cond, floor_cond]
titles = ['wall', 'floor', 'roof', 'water', 'electric', 'toilet', 'energy', 'rubbish', 'wall condition', 'roof condition', 'floor condition']
fig, ax =plt.subplots(len(col_types)//3 + 1, 3, figsize=(30,10))
plt.subplots_adjust(top=2.0)
 x=0
y=0
for i, cols in enumerate(col_types):
if(i>0)&(i % 3==0):
x=x+1
y=0
title = titles[i]
plot_df = pd.melt(cond_data, id_vars=['Target'], value_vars=cols ).groupby(["Target","variable"] ).apply(lambda x: np.mean(x))
plot_df["variable"] = plot_df.index.get_level_values(1)
 plot_df = plot_df.rename(columns = dict(value="Percentage Frequency", variable = "Attribute"))
sns.barplot(x="Attribute", y = "Percentage Frequency", hue="Target", data=plot_df, ax=ax[x,y])
 ax[x,y].title.set_text(title)
 y=y+1 
<categorify>
counting_number_of_values_dict(data,'spoken_languages' ).value_counts()
geoTrain= train.loc[train['parentesco1'] == 1, :].copy()
geoTrain['lugar'] = np.argmax(np.array(cond_data[["lugar1","lugar2","lugar3","lugar4","lugar5","lugar6"]]),
axis = 1)
 geoTrain['lugar']=geoTrain['lugar'].replace({5:6,4:5,3:4,2:3,1:2,0:1})
<define_variables>
languages = list(data['spoken_languages'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 Counter([i for j in languages for i in j] ).most_common(15 )
wall = ["paredblolad","paredzocalo","paredpreb","pareddes","paredmad","paredzinc","paredfibras","paredother"]
floor = ["pisomoscer","pisocemento","pisoother","pisonatur","pisonotiene","pisomadera"]
roof = ['techozinc','techoentrepiso','techocane','techootro', 'cielorazo']
water = ["abastaguadentro","abastaguafuera","abastaguano"]
electric = ["public","planpri","noelec","coopele"]
toilet = ['sanitario1','sanitario2','sanitario3','sanitario5','sanitario6']
energy = ['energcocinar1','energcocinar2','energcocinar3','energcocinar4']
rubbish = ['elimbasu1','elimbasu2','elimbasu3','elimbasu4','elimbasu5','elimbasu6']
wall_cond = ['epared1','epared2','epared3']
roof_cond = ['etecho1','etecho2','etecho3']
floor_cond = ['eviv1','eviv2','eviv3']
col_types = [wall, floor, roof, water, electric, toilet, energy, rubbish, wall_cond, roof_cond, floor_cond]
titles = ['wall', 'floor', 'roof', 'water', 'electric', 'toilet', 'energy', 'rubbish', 'wall condition', 'roof condition', 'floor condition']
fig, ax =plt.subplots(len(col_types)//3 + 1, 3, figsize=(30,10))
plt.subplots_adjust(top=2.0)
 x=0
y=0
for i, cols in enumerate(col_types):
if(i>0)&(i % 3==0):
x=x+1
y=0
title = titles[i]
plot_df = pd.melt(geoTrain, id_vars=['lugar'], value_vars=cols ).groupby(["lugar","variable"] ).apply(lambda x: np.mean(x))
plot_df["variable"] = plot_df.index.get_level_values(1)
 plot_df = plot_df.rename(columns = dict(value="Percentage Frequency", variable = "Attribute"))
sns.barplot(x="Attribute", y = "Percentage Frequency", hue="lugar", data=plot_df, ax=ax[x,y])
 ax[x,y].title.set_text(title)
 y=y+1 
<feature_engineering>
data['num_languages'] = counting_number_of_values_dict(data,'spoken_languages')
 data['all_languages'] = data['spoken_languages'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_languages = [m[0] for m in Counter([i for j in languages for i in j] ).most_common(30)]
for g in top_languages:
data['language_' + g] = data['all_languages'].apply(lambda x: 1 if g in x else 0 )
train['Adults']=train['hogar_adul']-train['hogar_mayor']
train['Dependents']=train['hogar_nin']+train['hogar_mayor']
train['Teenagers']=train['hogar_nin']-train['r4t1']
train['dependency']=train['Dependents']/train['Adults']
train['P_Adults']=train['Adults']/train['hogar_total']
train['P_Adults_Male']=train['r4h3']/train['hogar_total']
train['P_Adults_Female']=train['r4m3']/train['hogar_total']
train['P_Children']=train['hogar_nin']/train['hogar_total']
train['P_Seniors']=train['hogar_mayor']/train['hogar_total']
train['P_Teenagers']=train['Teenagers']/train['hogar_total']
train['RentHH']=train['v2a1']/train['hogar_total']
train['RentAdults'] = train['v2a1']/train['Adults']
train['RentAdults'] = train['RentAdults'].fillna(train['v2a1'])
 train['Tablet_PP']=train['v18q1']/train['hogar_total']
train['Phone_PP']=train['qmobilephone']/train['hogar_total']
train['Bedroom_PP']=train['bedrooms']/train['hogar_total']
train['Appliances']=train['refrig']+train['computer']+train['television']
train['Appliances_PP']=train['Appliances']/test['hogar_total']
train['HHS_Diff']=train['tamviv']-train['hhsize']
train["AdultsPerRoom"] = train['Adults']/train['rooms']
train["DependentsPerRoom"] = train['Dependents']/train['rooms']
train["TeenagersPerRoom"] = train['Teenagers']/train['rooms']
train["MalesPerRoom"] = train['r4h3']/train['rooms']
train["FemalesPerRoom"] = train['r4m3']/train['rooms']
train["BedroomPerRoom"] = train['bedrooms']/train['rooms']
train["RentPerRoom"] = train['v2a1']/train['rooms']
train["Schooling_Age"] = train["escolari"]/train["age"]
train['SchoolingProp'] = train['rez_esc']/train['escolari']
train["EduLevel"] = 0
train.loc[train["instlevel9"] == 1,"EduLevel"] = 6
train.loc[train["instlevel8"] == 1,"EduLevel"] = 5
train.loc[train["instlevel7"] == 1,"EduLevel"] = 3
train.loc[train["instlevel5"] == 1,"EduLevel"] = 2
train.loc[(train[["instlevel4","instlevel3","instlevel6"]].sum(axis = 1)> 0),"EduLevel"] = 1
train.head()
test['Adults']=test['hogar_adul']-test['hogar_mayor']
test['Dependents']=test['hogar_nin']+test['hogar_mayor']
test['Teenagers']=test['hogar_nin']-test['r4t1']
test['dependency']=test['Dependents']/test['Adults']
test['P_Adults']=test['Adults']/test['hogar_total']
test['P_Adults_Male']=test['r4h3']/test['hogar_total']
test['P_Adults_Female']=test['r4m3']/test['hogar_total']
test['P_Children']=test['hogar_nin']/test['hogar_total']
test['P_Seniors']=test['hogar_mayor']/test['hogar_total']
test['P_Teenagers']=test['Teenagers']/test['hogar_total']
test['RentHH']=test['v2a1']/test['hogar_total']
test['RentAdults']=test['v2a1']/test['Adults']
test['RentAdults'] = test['RentAdults'].fillna(test['v2a1'])
 test['Tablet_PP']=test['v18q1']/test['hogar_total']
test['Phone_PP']=test['qmobilephone']/test['hogar_total']
test['Bedroom_PP']=test['bedrooms']/test['hogar_total']
test['Appliances']=test['refrig']+test['computer']+test['television']
test['Appliances_PP']=test['Appliances']/test['hogar_total']
test['HHS_Diff']=test['tamviv']-test['hhsize']
test["AdultsPerRoom"] = test['Adults']/test['rooms']
test["DependentsPerRoom"] = test['Dependents']/test['rooms']
test["TeenagersPerRoom"] = test['Teenagers']/test['rooms']
test["MalesPerRoom"] = test['r4h3']/test['rooms']
test["FemalesPerRoom"] = test['r4m3']/test['rooms']
test["BedroomPerRoom"] = test['bedrooms']/test['rooms']
test["RentPerRoom"] = test['v2a1']/test['rooms']
test["Schooling_Age"] = test["escolari"]/test["age"]
test['SchoolingProp'] = test['rez_esc']/test['escolari']
test["EduLevel"] = 0
test.loc[test["instlevel9"] == 1,"EduLevel"] = 6
test.loc[test["instlevel8"] == 1,"EduLevel"] = 5
test.loc[test["instlevel7"] == 1,"EduLevel"] = 3
test.loc[test["instlevel5"] == 1,"EduLevel"] = 2
test.loc[(test[["instlevel4","instlevel3","instlevel6"]].sum(axis = 1)> 0),"EduLevel"] = 1
test.head()<create_dataframe>
data.drop(['spoken_languages', 'all_languages'], axis=1, inplace=True )
List_Mean = ['rez_esc', 'male', 'female', 'estadocivil1', 'estadocivil2', 'estadocivil3', 'estadocivil4', 'estadocivil5',
'estadocivil6', 'estadocivil7', 'parentesco2','parentesco3', 'parentesco4', 'parentesco5', 'parentesco6', 'parentesco7',
'parentesco8', 'parentesco9', 'parentesco10', 'parentesco11', 'parentesco12','instlevel1', 'instlevel2', 'instlevel3',
'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9','overcrowding']
List_Summary = ['age', 'escolari','dis','EduLevel', 'Schooling_Age']
trainGP = pd.DataFrame()
testGP = pd.DataFrame()
for item in List_Mean:
group_train_mean = train[item].groupby(train['idhogar'] ).mean()
group_test_mean = test[item].groupby(test['idhogar'] ).mean()
new_col = item + '_mean'
trainGP[new_col] = group_train_mean
testGP[new_col] = group_test_mean
for item in List_Summary:
for function in ['mean','std','min','max','sum']:
group_train = train[item].groupby(train['idhogar'] ).agg(function)
 group_test = test[item].groupby(test['idhogar'] ).agg(function)
 new_col = item + '_' + function
trainGP[new_col] = group_train
testGP[new_col] = group_test
trainGP['age_range']=trainGP['age_max']-trainGP['age_min']
testGP['age_range']=testGP['age_max']-testGP['age_min']
trainGP['escolari_range']=trainGP['escolari_max']-trainGP['escolari_min']
testGP['escolari_range']=testGP['escolari_max']-testGP['escolari_min']
trainGP["age_std"] = trainGP["age_std"].fillna(0)
 trainGP["escolari_std"] = trainGP["escolari_std"].fillna(0)
 trainGP["dis_std"] = trainGP["dis_std"].fillna(0)
 trainGP["EduLevel_std"] = trainGP["EduLevel_std"].fillna(0)
 trainGP["Schooling_Age_std"] = trainGP["Schooling_Age_std"].fillna(0)
 testGP["age_std"] = testGP["age_std"].fillna(0)
 testGP["escolari_std"] = testGP["escolari_std"].fillna(0)
 testGP["dis_std"] = testGP["dis_std"].fillna(0)
 testGP["EduLevel_std"] = testGP["EduLevel_std"].fillna(0)
 testGP["Schooling_Age_std"] = testGP["Schooling_Age_std"].fillna(0)
 trainGP.head()
testGP.head()<feature_engineering>
counting_number_of_values_dict(data,'production_countries' ).value_counts()
train["GoodCondition"] = train["paredblolad"] + \
train["pisomoscer"] + \
train["cielorazo"] + \
train["abastaguadentro"] + \
train["public"] + \
train["sanitario2"] + \
train["energcocinar2"] + \
train["elimbasu1"] + \
train["epared3"] + \
train["etecho3"] + \
train["eviv3"]
train["GoodCondition"] = train["GoodCondition"]/11
train["AverageCondition"] =(train["paredzocalo"] + train["paredpreb"])/2 + \
train["techozinc"] + \
train["abastaguafuera"] + \
train["coopele"] + \
train["sanitario3"] + \
train["energcocinar3"] + \
train["elimbasu2"] + \
train["epared2"] + \
train["etecho2"] + \
train["eviv2"]
train["AverageCondition"] = train["AverageCondition"]/10
train["BadCondition"] =(train["pareddes"] + train["paredmad"] + train["paredzinc"])/3 + \ 
(train["pisocemento"] + train["pisonotiene"] + train["pisomadera"])/3 + \
train["abastaguano"] + \
train["noelec"] + \ 
(train["sanitario1"] + train["sanitario5"])/2 + \ 
(train["energcocinar1"] + train["energcocinar4"])/2 + \
train["elimbasu3"] + \
train["epared1"] + \
train["etecho1"] + \
train["eviv1"]
train["BadCondition"] = train["BadCondition"]/10
test["GoodCondition"] = test["paredblolad"] + \
test["pisomoscer"] + \
test["cielorazo"] + \
test["abastaguadentro"] + \
test["public"] + \
test["sanitario2"] + \
test["energcocinar2"] + \
test["elimbasu1"] + \
test["epared3"] + \
test["etecho3"] + \
test["eviv3"]
test["GoodCondition"] = test["GoodCondition"]/11
test["AverageCondition"] =(test["paredzocalo"] + test["paredpreb"])/2 + \
test["techozinc"] + \
test["abastaguafuera"] + \
test["coopele"] + \
test["sanitario3"] + \
test["energcocinar3"] + \
test["elimbasu2"] + \
test["epared2"] + \
test["etecho2"] + \
test["eviv2"]
test["AverageCondition"] = test["AverageCondition"]/10
test["BadCondition"] =(test["pareddes"] + test["paredmad"] + test["paredzinc"])/3 + \ 
(test["pisocemento"] + test["pisonotiene"] + test["pisomadera"])/3 + \
test["abastaguano"] + \
test["noelec"] + \ 
(test["sanitario1"] +test["sanitario5"])/2 + \ 
(test["energcocinar1"] + test["energcocinar4"])/2 + \
test["elimbasu3"] + \
test["epared1"] + \
test["etecho1"] + \
test["eviv1"]
test["BadCondition"] = test["BadCondition"]/10<merge>
data['num_countries'] = counting_number_of_values_dict(data, 'production_countries')
 dict_one_hot_code(data,name_column= 'production_countries', name_new_column='production_country')
trainGP = trainGP.reset_index()
testGP = testGP.reset_index()
trainML = pd.merge(train, trainGP, on='idhogar')
 testML = pd.merge(test, testGP, on='idhogar')
 trainML = trainML.query("parentesco1==1")
 trainML.fillna(value=0, inplace=True)
 testML.fillna(value=0, inplace=True)
 trainML = trainML.replace([np.inf, -np.inf], 0)
 testML = testML.replace([np.inf, -np.inf], 0)
 train.shape, test.shape, trainGP.shape, testGP.shape, trainML.shape, testML.shape<count_missing_values>
data.drop('production_countries', axis=1, inplace=True )
trainML.isna().any().any() ,testML.isna().any().any()<create_dataframe>
counting_number_of_values_dict(data,'production_companies' ).value_counts()
submission = testML.copy() [['Id']]<drop_column>
data['prod_companies_total_number'] = counting_number_of_values_dict(data,'production_companies' )
trainML.drop(columns=['idhogar','Id','tamhog','r4t3','hhsize','hogar_adul','edjefe','edjefa'],inplace=True)
 testML.drop(columns=['idhogar','Id','tamhog','r4t3','hhsize','hogar_adul','edjefe','edjefa'],inplace=True)
<sort_values>
dict_one_hot_code(data,'production_companies','all_production_companies' )
correlation=trainML.corr()
correlation = correlation['Target'].sort_values(ascending=False)
 print(f'The most 20 positively correlated feature:
{correlation.head(20)}')
 print('*'*50)
 print(f'The most 20 negatively correlated feature:
{correlation.tail(20)}' )<drop_column>
data.drop('production_companies', axis=1, inplace=True )
columns_remove = ['elimbasu5','estadocivil1','parentesco1','parentesco2','parentesco3','parentesco4','parentesco5','parentesco6','parentesco7','parentesco8','parentesco9','parentesco10','parentesco11','parentesco12']
trainML.drop(columns=columns_remove, inplace=True)
 testML.drop(columns=columns_remove, inplace=True )<prepare_x_and_y>
data['homepage'].isnull().value_counts()
X = trainML.drop(columns=['Target'])
 y = trainML['Target']
Xtest = testML.copy() 
<train_model>
data['has_homepage'] = 0
data.loc[data['homepage'].isnull() == False, 'has_homepage'] = 1
data.drop('homepage', axis=1, inplace=True )
svm_model = SVC(gamma='auto')
 svm_model.fit(X, y )<predict_on_test>
data_language = pd.DataFrame(data.groupby('original_language')["revenue"].sum().sort_values(ascending=False ).head(10))
test_prediction_svm = svm_model.predict(Xtest )<train_model>
data['is_english_original_language'] = 0
data.loc[data['original_language'] == 'en', 'is_english_original_language'] = 1
rf_model = RandomForestClassifier(n_estimators=100)
 rf_model.fit(X,y)
<predict_on_test>
data.drop('original_language', axis=1, inplace=True )
test_prediction_rf = rf_model.predict(Xtest )<compute_train_metric>
data[['release_month','release_day','release_year']]=data['release_date'].str.split('/',expand=True ).replace(np.nan, -1 ).astype(int)
 data.loc[(data['release_year'] <= 19)&(data['release_year'] < 100), "release_year"] += 2000
data.loc[(data['release_year'] > 19)&(data['release_year'] < 100), "release_year"] += 1900
releaseDate = pd.to_datetime(data['release_date'])
 data['release_dayofweek'] = releaseDate.dt.dayofweek
data['release_quarter'] = releaseDate.dt.quarter
def evaluate_macroF1_lgb(truth, predictions):
predictions = np.resize(predictions, new_shape=(4,len(truth)))
 pred_labels = predictions.argmax(axis=0)
 f1 = f1_score(truth, pred_labels, average='macro')
 eval_name, val, is_higher_better = "macroF1", f1, True
return(eval_name, val, is_higher_better)
 clf = lgb.LGBMClassifier(max_depth=12, learning_rate=0.005, objective='multiclass',
random_state=69, silent=True, metric='None',
n_jobs=4, n_estimators=2500, class_weight='balanced',
colsample_bytree = 0.89, min_child_samples = 80, num_leaves = 14, subsample = 0.96)
 clf<choose_model_class>
data['title_length'] = data['original_title'].apply(lambda x: len(x)if x != '' else 0 )
kfold = 5
kf = StratifiedKFold(n_splits=kfold, shuffle=True, random_state = 123 )<train_model>
corr = data.corr()
corr['revenue'].sort_values(ascending=False ).head(20 )
val_predictions_logloss = []
test_predictions_logloss = []
test_predictions_logloss_proba = []
for train_index, val_index in kf.split(X, y):
print("=======")
 X_train, X_val = X.iloc[train_index], X.iloc[val_index]
y_train, y_val = y.iloc[train_index], y.iloc[val_index]
clf.fit(X_train, y_train, eval_set=[(X_val, y_val)], eval_metric="logloss",
early_stopping_rounds=400, verbose=200)
 val_predictions_logloss.append(clf.predict(X_val))
test_predictions_logloss.append(clf.predict(Xtest))
test_predictions_logloss_proba.append(clf.predict_proba(Xtest)) 
<train_model>
data.isnull().sum().sort_values(ascending=False )
X2 = X.iloc[:,top_100_indices]
Xtest2 = testML.copy().iloc[:,top_100_indices]
test_predictions2 = []
test_predictions_proba2 = []
for train_index, val_index in kf.split(X2, y):
print("=======")
 X_train, X_val = X2.iloc[train_index], X2.iloc[val_index]
y_train, y_val = y.iloc[train_index], y.iloc[val_index]
clf.fit(X_train, y_train, eval_set=[(X_val, y_val)], eval_metric="logloss",
early_stopping_rounds=400, verbose=200)
 test_predictions2.append(clf.predict(Xtest2))
test_predictions_proba2.append(clf.predict_proba(Xtest2)) 
<save_to_csv>
data.drop(['imdb_id', 'poster_path', 'release_date', 'status','budget_log','cast_character_','language_'], axis=1, inplace=True )
submission_svm = submission.copy()
submission_svm['Target'] = np.array(test_prediction_svm ).astype(int)
 submission_svm.to_csv("submission_01.csv", index=False)
 print(submission_svm['Target'].value_counts() /submission_svm['Target'].value_counts().sum())
 print("The Macro F1 Score(Test)is: 0.28218" )<save_to_csv>
data_dropping_names = data.drop(['original_title','overview','tagline','title'], axis=1)
 train = data_dropping_names[data_dropping_names['source'] == 'train'].copy()
test = data_dropping_names[data_dropping_names['source'] == 'test'].copy()
train_labels = train['revenue_log']
train.drop(['id', 'revenue', 'source', 'revenue_log'], axis=1, inplace=True)
 test_final = test.drop(['id', 'source','revenue','revenue_log'], axis=1 )
submission_rf = submission.copy()
submission_rf['Target'] = np.array(test_prediction_rf ).astype(int)
 submission_rf.to_csv("submission_02.csv", index=False)
 print(submission_rf['Target'].value_counts() /submission_rf['Target'].value_counts().sum())
 print("The Macro F1 Score(Test)is: 0.37434" )<save_to_csv>
train.isnull().sum()
submission_full_logloss = submission.copy()
voted_test_predictions = np.apply_along_axis(lambda x: np.bincount(x ).argmax() , axis=0, arr=test_predictions_logloss)
 submission_full_logloss['Target'] = np.array(voted_test_predictions ).astype(int)
 submission_full_logloss.to_csv("submission_03.csv", index=False)
 print(submission_full_logloss['Target'].value_counts() /submission_full_logloss['Target'].value_counts().sum())
 print("The Macro F1 Score(Test)is: 0.43970" )<save_to_csv>
train.isnull().sum()
submission_top100_logloss = submission.copy()
voted_test_predictions = np.apply_along_axis(lambda x: np.bincount(x ).argmax() , axis=0, arr=test_predictions2)
 submission_top100_logloss['Target'] = np.array(voted_test_predictions ).astype(int)
 submission_top100_logloss.to_csv("submission_04.csv", index=False)
 print(submission_top100_logloss['Target'].value_counts() /submission_top100_logloss['Target'].value_counts().sum())
 print("The Macro F1 Score(Test)is: 0.44019" )<set_options>
num_pipeline = Pipeline([ 
('imputer', SimpleImputer(strategy="median")) , 
('robust_scaler', RobustScaler()),
] )
%matplotlib inline
sns.set() 
<load_from_csv>
n_fold = 10
folds = KFold(n_splits=n_fold, shuffle=True, random_state=42)
 def train_model(X, X_test, y, params=None, folds=folds, model_type='lgb', plot_feature_importance=False, model=None):
prediction = np.zeros(X_test.shape[0])
 scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(folds.split(X)) :
print('Fold', fold_n, 'started at', time.ctime())
 if model_type == 'sklearn':
X_train, X_valid = X[train_index], X[valid_index]
else:
X_train, X_valid = X.values[train_index], X.values[valid_index]
y_train, y_valid = y[train_index], y[valid_index]
if model_type == 'lgb':
model = lgb.LGBMRegressor(**params, n_estimators = 20000, nthread = 4, n_jobs = -1)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='rmse',
verbose=1000, early_stopping_rounds=200)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration_)
 if model_type == 'xgb':
train_data = xgb.DMatrix(data=X_train, label=y_train)
 valid_data = xgb.DMatrix(data=X_valid, label=y_valid)
 watchlist = [(train_data, 'train'),(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=20000, evals=watchlist, early_stopping_rounds=200, verbose_eval=500, params=params)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid), ntree_limit=model.best_ntree_limit)
 y_pred = model.predict(xgb.DMatrix(X_test.values), ntree_limit=model.best_ntree_limit)
 if model_type == 'sklearn':
model = model
model.fit(X_train, y_train)
 y_pred_valid = model.predict(X_valid ).reshape(-1,)
 score = mean_squared_error(y_valid, y_pred_valid)
 y_pred = model.predict(X_test)
 if model_type == 'cat':
model = CatBoostRegressor(iterations=20000, eval_metric='RMSE', **params)
 model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=False)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test)
 scores.append(mean_squared_error(y_valid, y_pred_valid)** 0.5)
 prediction += y_pred
if model_type == 'lgb':
fold_importance = pd.DataFrame()
fold_importance["feature"] = X.columns
fold_importance["importance"] = model.feature_importances_
fold_importance["fold"] = fold_n + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 prediction /= n_fold
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 gc.enable()
del model, y_pred_valid, X_test,X_train,X_valid, y_pred, y_train
gc.collect()
if model_type == 'lgb':
feature_importance["importance"] /= n_fold
if plot_feature_importance:
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
return scores, prediction, feature_importance
return scores, prediction
else:
return scores, prediction
def import_train_test() :
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 return train, test
train, test = import_train_test()<count_unique_values>
train_dummies = pd.get_dummies(train)
 test_dummies = pd.get_dummies(test)
 train_dummies, test_dummies = train_dummies.align(test_dummies, axis=1, join='inner' )
print(f"There are {len(train.idhogar.unique())} unique households in the training set.")
 print(f"There are {len(test.idhogar.unique())} unique households in the test set." )<feature_engineering>
params = {
'num_leaves': 30,
'min_data_in_leaf': 20,
'objective': 'regression',
'max_depth': 6,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.2,
}
score_lgb, prediction_lgb, _ = train_model(train_dummies, test_dummies, train_labels, params=params, model_type='lgb', plot_feature_importance=True )
q1s = train[numeric_non_boolean_attributes].quantile(0.25)
 q3s = train[numeric_non_boolean_attributes].quantile(0.75)
 iqrs = pd.DataFrame([q1s, q3s], index=['q1', 'q3'] ).transpose().drop('Target')
 iqrs['iqr'] = iqrs.q3 - iqrs.q1
iqrs['iqr_amplified'] = iqrs.iqr * 1.5
iqrs['outlier_min'] = iqrs.q1 - iqrs.iqr_amplified
iqrs['outlier_max'] = iqrs.q3 + iqrs.iqr_amplified
def min_outlier_count(iqr_row):
return len(train[train[iqr_row.name] < iqrs.loc[[iqr_row.name]].outlier_min[0]])
 def max_outlier_count(iqr_row):
return len(train[train[iqr_row.name] > iqrs.loc[[iqr_row.name]].outlier_max[0]])
 iqrs['min_outlier_count'] = iqrs.apply(min_outlier_count, axis=1)
 iqrs['max_outlier_count'] = iqrs.apply(max_outlier_count, axis=1)
 iqrs[(iqrs.min_outlier_count > 0)|(iqrs.max_outlier_count > 0)]
with pd.option_context('display.max_columns', len(iqrs)) :
print(train[iqrs.index].describe() )<define_variables>
score_lgb.sort(reverse=True)
 dictvalues = {'RMSE_lgb': score_lgb}
id_features = ['Id', 'idhogar']
features_and_descriptions = [('v2a1', 'Monthly rent payment'), 
('hacdor', '=1 Overcrowding by bedrooms'), 
('rooms', 'number of all rooms in the house'), 
('hacapo', '=1 Overcrowding by rooms'), 
('v14a', '=1 has bathroom in the household'), 
('refrig', '=1 if the household has refrigerator'), 
('v18q', 'owns a tablet'), 
('v18q1', 'number of tablets household owns'), 
('r4h1', 'Males younger than 12 years of age'), 
('r4h2', 'Males 12 years of age and older'), 
('r4h3', 'Total males in the household'), 
('r4m1', 'Females younger than 12 years of age'), 
('r4m2', 'Females 12 years of age and older'), 
('r4m3', 'Total females in the household'), 
('r4t1', 'persons younger than 12 years of age'), 
('r4t2', 'persons 12 years of age and older'), 
('r4t3', 'Total persons in the household'), 
('tamhog', 'size of the household'), 
('tamviv', 'number of persons living in the household'), 
('escolari', 'years of schooling'), 
('rez_esc', 'Years behind in school'), 
('hhsize', 'household size'), 
('paredblolad', '=1 if predominant material on the outside wall is block or brick'), 
('paredzocalo', '"=1 if predominant material on the outside wall is socket(wood, zinc or absbesto"'), 
('paredpreb', '=1 if predominant material on the outside wall is prefabricated or cement'), 
('pareddes', '=1 if predominant material on the outside wall is waste material'), 
('paredmad', '=1 if predominant material on the outside wall is wood'), 
('paredzinc', '=1 if predominant material on the outside wall is zink'), 
('paredfibras', '=1 if predominant material on the outside wall is natural fibers'), 
('paredother', '=1 if predominant material on the outside wall is other'), 
('pisomoscer', '"=1 if predominant material on the floor is mosaic, ceramic, terrazo"'), 
('pisocemento', '=1 if predominant material on the floor is cement'), 
('pisoother', '=1 if predominant material on the floor is other'), 
('pisonatur', '=1 if predominant material on the floor is natural material'), 
('pisonotiene', '=1 if no floor at the household'), 
('pisomadera', '=1 if predominant material on the floor is wood'), 
('techozinc', '=1 if predominant material on the roof is metal foil or zink'), 
('techoentrepiso', '"=1 if predominant material on the roof is fiber cement, mezzanine "'), 
('techocane', '=1 if predominant material on the roof is natural fibers'), 
('techootro', '=1 if predominant material on the roof is other'), 
('cielorazo', '=1 if the house has ceiling'), 
('abastaguadentro', '=1 if water provision inside the dwelling'), 
('abastaguafuera', '=1 if water provision outside the dwelling'), 
('abastaguano', '=1 if no water provision'), 
('public', '"=1 electricity from CNFL, ICE, ESPH/JASEC"'), 
('planpri', '=1 electricity from private plant'), 
('noelec', '=1 no electricity in the dwelling'), 
('coopele', '=1 electricity from cooperative'), 
('sanitario1', '=1 no toilet in the dwelling'), 
('sanitario2', '=1 toilet connected to sewer or cesspool'), 
('sanitario3', '=1 toilet connected to septic tank'), 
('sanitario5', '=1 toilet connected to black hole or letrine'), 
('sanitario6', '=1 toilet connected to other system'), 
('energcocinar1', '=1 no main source of energy used for cooking(no kitchen)'), 
('energcocinar2', '=1 main source of energy used for cooking electricity'), 
('energcocinar3', '=1 main source of energy used for cooking gas'), 
('energcocinar4', '=1 main source of energy used for cooking wood charcoal'), 
('elimbasu1', '=1 if rubbish disposal mainly by tanker truck'), 
('elimbasu2', '=1 if rubbish disposal mainly by botan hollow or buried'), 
('elimbasu3', '=1 if rubbish disposal mainly by burning'), 
('elimbasu4', '=1 if rubbish disposal mainly by throwing in an unoccupied space'), 
('elimbasu5', '"=1 if rubbish disposal mainly by throwing in river, creek or sea"'), 
('elimbasu6', '=1 if rubbish disposal mainly other'), 
('epared1', '=1 if walls are bad'), 
('epared2', '=1 if walls are regular'), 
('epared3', '=1 if walls are good'), 
('etecho1', '=1 if roof are bad'), 
('etecho2', '=1 if roof are regular'), 
('etecho3', '=1 if roof are good'), 
('eviv1', '=1 if floor are bad'), 
('eviv2', '=1 if floor are regular'), 
('eviv3', '=1 if floor are good'), 
('dis', '=1 if disable person'), 
('male', '=1 if male'), 
('female', '=1 if female'), 
('estadocivil1', '=1 if less than 10 years old'), 
('estadocivil2', '=1 if free or coupled uunion'), 
('estadocivil3', '=1 if married'), 
('estadocivil4', '=1 if divorced'), 
('estadocivil5', '=1 if separated'), 
('estadocivil6', '=1 if widow/er'), 
('estadocivil7', '=1 if single'), 
('parentesco1', '=1 if household head'), 
('parentesco2', '=1 if spouse/partner'), 
('parentesco3', '=1 if son/doughter'), 
('parentesco4', '=1 if stepson/doughter'), 
('parentesco5', '=1 if son/doughter in law'), 
('parentesco6', '=1 if grandson/doughter'), 
('parentesco7', '=1 if mother/father'), 
('parentesco8', '=1 if father/mother in law'), 
('parentesco9', '=1 if brother/sister'), 
('parentesco10', '=1 if brother/sister in law'), 
('parentesco11', '=1 if other family member'), 
('parentesco12', '=1 if other non family member'), 
('idhogar', 'Household level identifier'), 
('hogar_nin', 'Number of children 0 to 19 in household'), 
('hogar_adul', 'Number of adults in household'), 
('hogar_mayor', ' 
('hogar_total', ' 
('dependency', 'Dependency rate, calculated =(number of members of the household younger than 19 or older than 64)/(number of member of household between 19 and 64)'), 
('edjefe', 'years of education of male head of household, based on the interaction of escolari(years of education), head of household and gender, yes=1 and no=0'), 
('edjefa', 'years of education of female head of household, based on the interaction of escolari(years of education), head of household and gender, yes=1 and no=0'), 
('meaneduc', 'average years of education for adults(18+)'), 
('instlevel1', '=1 no level of education'), 
('instlevel2', '=1 incomplete primary'), 
('instlevel3', '=1 complete primary'), 
('instlevel4', '=1 incomplete academic secondary level'), 
('instlevel5', '=1 complete academic secondary level'), 
('instlevel6', '=1 incomplete technical secondary level'), 
('instlevel7', '=1 complete technical secondary level'), 
('instlevel8', '=1 undergraduate and higher education'), 
('instlevel9', '=1 postgraduate higher education'), 
('bedrooms', 'number of bedrooms'), 
('overcrowding', ' 
('tipovivi1', '=1 own and fully paid house'), 
('tipovivi2', '"=1 own, paying in installments"'), 
('tipovivi3', '=1 rented'), 
('tipovivi4', '=1 precarious'), 
('tipovivi5', '"=1 other(assigned, borrowed)"'), 
('computer', '=1 if the household has notebook or desktop computer'), 
('television', '=1 if the household has TV'), 
('mobilephone', '=1 if mobile phone'), 
('qmobilephone', ' 
('lugar1', '=1 region Central'), 
('lugar2', '=1 region Chorotega'), 
('lugar3', '=1 region PacÃƒÂ­fico central'), 
('lugar4', '=1 region Brunca'), 
('lugar5', '=1 region Huetar AtlÃƒÂ¡ntica'), 
('lugar6', '=1 region Huetar Norte'), 
('area1', '=1 zona urbana'), 
('area2', '=2 zona rural'), 
('age', 'Age in years'), 
('SQBescolari', 'escolari squared'), 
('SQBage', 'age squared'), 
('SQBhogar_total', 'hogar_total squared'), 
('SQBedjefe', 'edjefe squared'), 
('SQBhogar_nin', 'hogar_nin squared'), 
('SQBovercrowding', 'overcrowding squared'), 
('SQBdependency', 'dependency squared'), 
('SQBmeaned', 'square of the mean years of education of adults(>=18)in the household'), 
('agesq', 'Age squared')]
all_features = [feature for feature, description in features_and_descriptions]
household_features_and_descriptions = [('v2a1', 'Monthly rent payment'), 
('hacdor', '=1 Overcrowding by bedrooms'), 
('rooms', 'number of all rooms in the house'), 
('hacapo', '=1 Overcrowding by rooms'), 
('v14a', '=1 has bathroom in the household'), 
('refrig', '=1 if the household has refrigerator'), 
('v18q1', 'number of tablets household owns'), 
('r4h1', 'Males younger than 12 years of age'), 
('r4h2', 'Males 12 years of age and older'), 
('r4h3', 'Total males in the household'), 
('r4m1', 'Females younger than 12 years of age'), 
('r4m2', 'Females 12 years of age and older'), 
('r4m3', 'Total females in the household'), 
('r4t1', 'persons younger than 12 years of age'), 
('r4t2', 'persons 12 years of age and older'), 
('r4t3', 'Total persons in the household'), 
('tamhog', 'size of the household'), 
('tamviv', 'number of persons living in the household'), 
('hhsize', 'household size'), 
('paredblolad', '=1 if predominant material on the outside wall is block or brick'), 
('paredzocalo', '"=1 if predominant material on the outside wall is socket(wood, zinc or absbesto"'), 
('paredpreb', '=1 if predominant material on the outside wall is prefabricated or cement'), 
('pareddes', '=1 if predominant material on the outside wall is waste material'), 
('paredmad', '=1 if predominant material on the outside wall is wood'), 
('paredzinc', '=1 if predominant material on the outside wall is zink'), 
('paredfibras', '=1 if predominant material on the outside wall is natural fibers'), 
('paredother', '=1 if predominant material on the outside wall is other'), 
('pisomoscer', '"=1 if predominant material on the floor is mosaic, ceramic, terrazo"'), 
('pisocemento', '=1 if predominant material on the floor is cement'), 
('pisoother', '=1 if predominant material on the floor is other'), 
('pisonatur', '=1 if predominant material on the floor is natural material'), 
('pisonotiene', '=1 if no floor at the household'), 
('pisomadera', '=1 if predominant material on the floor is wood'), 
('techozinc', '=1 if predominant material on the roof is metal foil or zink'), 
('techoentrepiso', '"=1 if predominant material on the roof is fiber cement, mezzanine "'), 
('techocane', '=1 if predominant material on the roof is natural fibers'), 
('techootro', '=1 if predominant material on the roof is other'), 
('cielorazo', '=1 if the house has ceiling'), 
('abastaguadentro', '=1 if water provision inside the dwelling'), 
('abastaguafuera', '=1 if water provision outside the dwelling'), 
('abastaguano', '=1 if no water provision'), 
('public', '"=1 electricity from CNFL, ICE, ESPH/JASEC"'), 
('planpri', '=1 electricity from private plant'), 
('noelec', '=1 no electricity in the dwelling'), 
('coopele', '=1 electricity from cooperative'), 
('sanitario1', '=1 no toilet in the dwelling'), 
('sanitario2', '=1 toilet connected to sewer or cesspool'), 
('sanitario3', '=1 toilet connected to septic tank'), 
('sanitario5', '=1 toilet connected to black hole or letrine'), 
('sanitario6', '=1 toilet connected to other system'), 
('energcocinar1', '=1 no main source of energy used for cooking(no kitchen)'), 
('energcocinar2', '=1 main source of energy used for cooking electricity'), 
('energcocinar3', '=1 main source of energy used for cooking gas'), 
('energcocinar4', '=1 main source of energy used for cooking wood charcoal'), 
('elimbasu1', '=1 if rubbish disposal mainly by tanker truck'), 
('elimbasu2', '=1 if rubbish disposal mainly by botan hollow or buried'), 
('elimbasu3', '=1 if rubbish disposal mainly by burning'), 
('elimbasu4', '=1 if rubbish disposal mainly by throwing in an unoccupied space'), 
('elimbasu5', '"=1 if rubbish disposal mainly by throwing in river, creek or sea"'), 
('elimbasu6', '=1 if rubbish disposal mainly other'), 
('epared1', '=1 if walls are bad'), 
('epared2', '=1 if walls are regular'), 
('epared3', '=1 if walls are good'), 
('etecho1', '=1 if roof are bad'), 
('etecho2', '=1 if roof are regular'), 
('etecho3', '=1 if roof are good'), 
('eviv1', '=1 if floor are bad'), 
('eviv2', '=1 if floor are regular'), 
('eviv3', '=1 if floor are good'), 
('hogar_nin', 'Number of children 0 to 19 in household'), 
('hogar_adul', 'Number of adults in household'), 
('hogar_mayor', ' 
('hogar_total', ' 
('dependency', 'Dependency rate, calculated =(number of members of the household younger than 19 or older than 64)/(number of member of household between 19 and 64)'), 
('edjefe', 'years of education of male head of household, based on the interaction of escolari(years of education), head of household and gender, yes=1 and no=0'), 
('edjefa', 'years of education of female head of household, based on the interaction of escolari(years of education), head of household and gender, yes=1 and no=0'), 
('meaneduc', 'average years of education for adults(18+)'), 
('bedrooms', 'number of bedrooms'), 
('overcrowding', ' 
('tipovivi1', '=1 own and fully paid house'), 
('tipovivi2', '"=1 own, paying in installments"'), 
('tipovivi3', '=1 rented'), 
('tipovivi4', '=1 precarious'), 
('tipovivi5', '"=1 other(assigned, borrowed)"'), 
('computer', '=1 if the household has notebook or desktop computer'), 
('television', '=1 if the household has TV'), 
('mobilephone', '=1 if mobile phone'), 
('qmobilephone', ' 
('lugar1', '=1 region Central'), 
('lugar2', '=1 region Chorotega'), 
('lugar3', '=1 region PacÃƒÂ­fico central'), 
('lugar4', '=1 region Brunca'), 
('lugar5', '=1 region Huetar AtlÃƒÂ¡ntica'), 
('lugar6', '=1 region Huetar Norte'), 
('area1', '=1 zona urbana'), 
('area2', '=2 zona rural')]
household_features = [feature for feature, description in household_features_and_descriptions]
individual_features_and_descriptions = [('v18q', 'owns a tablet'), 
('escolari', 'years of schooling'), 
('rez_esc', 'Years behind in school'), 
('dis', '=1 if disable person'), 
('male', '=1 if male'), 
('female', '=1 if female'), 
('estadocivil1', '=1 if less than 10 years old'), 
('estadocivil2', '=1 if free or coupled uunion'), 
('estadocivil3', '=1 if married'), 
('estadocivil4', '=1 if divorced'), 
('estadocivil5', '=1 if separated'), 
('estadocivil6', '=1 if widow/er'), 
('estadocivil7', '=1 if single'), 
('parentesco1', '=1 if household head'), 
('parentesco2', '=1 if spouse/partner'), 
('parentesco3', '=1 if son/doughter'), 
('parentesco4', '=1 if stepson/doughter'), 
('parentesco5', '=1 if son/doughter in law'), 
('parentesco6', '=1 if grandson/doughter'), 
('parentesco7', '=1 if mother/father'), 
('parentesco8', '=1 if father/mother in law'), 
('parentesco9', '=1 if brother/sister'), 
('parentesco10', '=1 if brother/sister in law'), 
('parentesco11', '=1 if other family member'), 
('parentesco12', '=1 if other non family member'), 
('instlevel1', '=1 no level of education'), 
('instlevel2', '=1 incomplete primary'), 
('instlevel3', '=1 complete primary'), 
('instlevel4', '=1 incomplete academic secondary level'), 
('instlevel5', '=1 complete academic secondary level'), 
('instlevel6', '=1 incomplete technical secondary level'), 
('instlevel7', '=1 complete technical secondary level'), 
('instlevel8', '=1 undergraduate and higher education'), 
('instlevel9', '=1 postgraduate higher education'), 
('age', 'Age in years')]
individual_features = [feature for feature, description in individual_features_and_descriptions]
squared_features_and_descriptions = [('SQBescolari', 'escolari squared'), 
('SQBage', 'age squared'), 
('SQBhogar_total', 'hogar_total squared'), 
('SQBedjefe', 'edjefe squared'), 
('SQBhogar_nin', 'hogar_nin squared'), 
('SQBovercrowding', 'overcrowding squared'), 
('SQBdependency', 'dependency squared'), 
('SQBmeaned', 'square of the mean years of education of adults(>=18)in the household'), 
('agesq', 'Age squared')]
squared_features = [feature for feature, description in squared_features_and_descriptions]
squared_household_features_and_descriptions = [('SQBhogar_total', 'hogar_total squared'), 
('SQBedjefe', 'edjefe squared'), 
('SQBhogar_nin', 'hogar_nin squared'), 
('SQBovercrowding', 'overcrowding squared'), 
('SQBdependency', 'dependency squared'), 
('SQBmeaned', 'square of the mean years of education of adults(>=18)in the household')]
squared_household_features = [feature for feature, description in squared_household_features_and_descriptions]
squared_individual_features_and_descriptions = [('SQBescolari', 'escolari squared'), 
('SQBage', 'age squared'), 
('agesq', 'Age squared')]
squared_individual_features = [feature for feature, description in squared_individual_features_and_descriptions]<drop_column>
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['revenue'] = np.expm1(prediction_lgb)
 sub.to_csv("lgb_model.csv", index=False )
for df, name in [(train, 'train'),(test, 'test')]:
assert df.agesq.equals(df.SQBage), f"agesq is not equivalent with SQBage in the {name} set"
lists_with_agesq = [features_and_descriptions, squared_features_and_descriptions, squared_individual_features_and_descriptions]
for feature_list in lists_with_agesq:
try:
feature_list.remove(( 'agesq', 'Age squared'))
except:
continue
for feature_list in lists_with_agesq:
assert 'agesq' not in {x for x, y in feature_list}, 'Duplicated column agesq is still in feature list.'<count_missing_values>
score_xgb.sort(reverse=True)
 dictvalues.update({'RMSE_XGB': score_xgb} )
def print_nan_counts(df):
nan_counts = df.isna().sum()
print(nan_counts[nan_counts > 0].sort_values(ascending = False))
print_nan_counts(train )<categorify>
sub['revenue'] = np.expm1(prediction_xgb)
 sub.to_csv("XGB_model.csv", index=False )
class ZerofillRezEscOutOfBounds(BaseEstimator, TransformerMixin):

 def __init__(self):
pass
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X.loc[X.rez_esc.isna() &(( X.age < 7)|(X.age > 19)) , 'rez_esc'] = 0
return X
zerofiller = ZerofillRezEscOutOfBounds()
train_zerofilled = zerofiller.transform(train.copy())
 assert not train_zerofilled[(train_zerofilled.age < 7)|(train_zerofilled.age > 19)].rez_esc.isna().any() , 'There are individuals younger than 7 or older than 19 with a na value for rez_esc.'
print(f"There are {train_zerofilled.rez_esc.isna().sum() } individuals with a na value for rez_esc.")
 print(f"There are {train_zerofilled[(train_zerofilled.age < 7)|(train_zerofilled.age > 19)].rez_esc.isna().sum() } individuals younger than 7 or older than 19 with a na value for rez_esc." )<categorify>
sub['revenue'] = np.expm1(prediction_cat)
 sub.to_csv("cat_model.csv", index=False )
class ZeroMaxRezEsc(BaseEstimator, TransformerMixin):

 def __init__(self):
pass
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X.loc[X.rez_esc > 5, 'rez_esc'] = 0
return X
zero_rez_esc = ZeroMaxRezEsc()
test_zero_rez_esc = zero_rez_esc.transform(test.copy())
 assert len(test_zero_rez_esc[test_zero_rez_esc.rez_esc > 5])== 0, 'There are individuals with greater than 5 rez_esc'<count_values>
sub['revenue'] = np.expm1(( prediction_lgb + prediction_xgb + prediction_cat)/ 3)
 sub.to_csv("combined.csv", index=False )
train.v18q.value_counts()<count_values>
from keras.models import Sequential, Model
from keras.optimizers import Adam, SGD
from keras.layers import Dense, Activation, Dropout, Input, concatenate,BatchNormalization
from keras.wrappers.scikit_learn import KerasRegressor
from keras.regularizers import l2
from keras.callbacks import EarlyStopping
print(train.v18q1.value_counts())
 print(len(train.loc[(train.v18q == 0 & train.v18q1.isna())]))<categorify>
def build_model(input_shape,n_hidden=10, n_neurons=30, optimizer = SGD(3e-3,momentum=0.9)) :
model = Sequential()
options = {"input_shape": input_shape}
for layer in range(n_hidden):
BatchNormalization()
model.add(Dense(n_neurons, activation="relu",
kernel_regularizer=l2(0.01), **options))
options = {}
BatchNormalization()
model.add(Dense(1, **options))
model.compile(loss="mean_squared_error", optimizer=optimizer)
 return model
class ZerofillV18q1ForFalseV18Q(BaseEstimator, TransformerMixin):

 def __init__(self):
pass
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X.loc[X.v18q == 0, 'v18q1'] = 0
return X
zerofiller = ZerofillV18q1ForFalseV18Q()
train_zerofilled = zerofiller.transform(train.copy())
 assert not train_zerofilled.v18q1.isna().any() , 'There are individuals with na value for v18q1.'<count_values>
train_dummies_prepared = num_pipeline.fit_transform(train_dummies )
for v in ['tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5']:
print(f"Value counts for {v} for individuals with missing v2a1:")
 print(train.loc[train.v2a1.isna() , v].value_counts())
 print('' )<prepare_x_and_y>
model_NN = build_model(input_shape= train_dummies_prepared.shape[1:], n_hidden=6,optimizer=Adam(3e-5))
class ZerofillV2a1(BaseEstimator, TransformerMixin):

 def __init__(self):
pass
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X.loc[(X.v2a1.isna())&(( X.tipovivi1 == 1)|(X.tipovivi5 == 1)) , 'v2a1'] = 0
return X
zerofiller = ZerofillV2a1()
train_zerofilled = zerofiller.transform(train.copy())
 assert not train_zerofilled[train_zerofilled.tipovivi4 == 0].v2a1.isna().any() , 'There are individuals with na value for v2a1.'<categorify>
model_history = model_NN.fit(train_dummies_prepared, train_labels, epochs=2500, batch_size=5, validation_split=0.1,callbacks=[EarlyStopping(patience=100)] )