Kaggle competition: TMDB Box Office Prediction
<SOS> metric: RMSLE Kaggle data source: tmdb-box-office-prediction<categorify>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import datetime as dt
from wordcloud import WordCloud, STOPWORDS
from collections import OrderedDict
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score
train['num_over_18'] = 0
train['num_over_18'] = train[train.age >= 18].groupby('idhogar' ).transform("count")
 train['num_over_18'] = train.groupby("idhogar")["num_over_18"].transform("max")
 train['num_over_18'] = train['num_over_18'].fillna(0)
 test['num_over_18'] = 0
test['num_over_18'] = test[test.age >= 18].groupby('idhogar' ).transform("count")
 test['num_over_18'] = test.groupby("idhogar")["num_over_18"].transform("max")
 test['num_over_18'] = test['num_over_18'].fillna(0)
 def extract_features(df):
df['bedrooms_to_rooms'] = df['bedrooms']/df['rooms']
df['rent_to_rooms'] = df['v2a1']/df['rooms']
df['tamhog_to_rooms'] = df['tamhog']/df['rooms']
df['r4t3_to_tamhog'] = df['r4t3']/df['tamhog']
df['r4t3_to_rooms'] = df['r4t3']/df['rooms']
df['v2a1_to_r4t3'] = df['v2a1']/df['r4t3']
df['v2a1_to_r4t3'] = df['v2a1']/(df['r4t3'] - df['r4t1'])
 df['hhsize_to_rooms'] = df['hhsize']/df['rooms']
df['rent_to_hhsize'] = df['v2a1']/df['hhsize']
df['rent_to_over_18'] = df['v2a1']/df['num_over_18']
df.loc[df.num_over_18 == 0, "rent_to_over_18"] = df[df.num_over_18 == 0].v2a1
df['dependency_yes'] = df['dependency'].apply(lambda x: 1 if x == 'yes' else 0)
 df['dependency_no'] = df['dependency'].apply(lambda x: 1 if x == 'no' else 0)
 extract_features(train)
 extract_features(test )<drop_column>
train = pd.read_csv(".. /input/tmdb-box-office-prediction/train.csv")
 test=pd.read_csv(".. /input/tmdb-box-office-prediction/test.csv" )
needless_cols = ['r4t3', 'tamhog', 'tamviv', 'hhsize', 'v18q', 'v14a', 'agesq',
'mobilephone', 'female', ]
instlevel_cols = [s for s in train.columns.tolist() if 'instlevel' in s]
needless_cols.extend(instlevel_cols)
 train = train.drop(needless_cols, axis=1)
 test = test.drop(needless_cols, axis=1 )<split>
data.loc[data['id'] == 16,'revenue'] = 192864
data.loc[data['id'] == 90,'budget'] = 30000000
data.loc[data['id'] == 118,'budget'] = 60000000
data.loc[data['id'] == 149,'budget'] = 18000000
data.loc[data['id'] == 313,'revenue'] = 12000000
data.loc[data['id'] == 451,'revenue'] = 12000000
data.loc[data['id'] == 464,'budget'] = 20000000
data.loc[data['id'] == 470,'budget'] = 13000000
data.loc[data['id'] == 513,'budget'] = 930000
data.loc[data['id'] == 797,'budget'] = 8000000
data.loc[data['id'] == 819,'budget'] = 90000000
data.loc[data['id'] == 850,'budget'] = 90000000
data.loc[data['id'] == 1007,'budget'] = 2
data.loc[data['id'] == 1112,'budget'] = 7500000
data.loc[data['id'] == 1131,'budget'] = 4300000
data.loc[data['id'] == 1359,'budget'] = 10000000
data.loc[data['id'] == 1542,'budget'] = 1
data.loc[data['id'] == 1570,'budget'] = 15800000
data.loc[data['id'] == 1571,'budget'] = 4000000
data.loc[data['id'] == 1714,'budget'] = 46000000
data.loc[data['id'] == 1721,'budget'] = 17500000
data.loc[data['id'] == 1865,'revenue'] = 25000000
data.loc[data['id'] == 1885,'budget'] = 12
data.loc[data['id'] == 2091,'budget'] = 10
data.loc[data['id'] == 2268,'budget'] = 17500000
data.loc[data['id'] == 2491,'budget'] = 6
data.loc[data['id'] == 2602,'budget'] = 31000000
data.loc[data['id'] == 2612,'budget'] = 15000000
data.loc[data['id'] == 2696,'budget'] = 10000000
data.loc[data['id'] == 2801,'budget'] = 10000000
data.loc[data['id'] == 335,'budget'] = 2
data.loc[data['id'] == 348,'budget'] = 12
data.loc[data['id'] == 470,'budget'] = 13000000
data.loc[data['id'] == 513,'budget'] = 1100000
data.loc[data['id'] == 640,'budget'] = 6
data.loc[data['id'] == 696,'budget'] = 1
data.loc[data['id'] == 797,'budget'] = 8000000
data.loc[data['id'] == 850,'budget'] = 1500000
data.loc[data['id'] == 1199,'budget'] = 5
data.loc[data['id'] == 1282,'budget'] = 9
data.loc[data['id'] == 1347,'budget'] = 1
data.loc[data['id'] == 1755,'budget'] = 2
data.loc[data['id'] == 1801,'budget'] = 5
data.loc[data['id'] == 1918,'budget'] = 592
data.loc[data['id'] == 2033,'budget'] = 4
data.loc[data['id'] == 2118,'budget'] = 344
data.loc[data['id'] == 2252,'budget'] = 130
data.loc[data['id'] == 2256,'budget'] = 1
data.loc[data['id'] == 2696,'budget'] = 10000000
def split_data(train, y, households, test_percentage=0.20, seed=None):
np.random.seed(seed=seed)
 train2 = train.copy()
cv_hhs = np.random.choice(households, size=int(len(households)* test_percentage), replace=False)
 cv_idx = np.isin(households, cv_hhs)
 X_test = train2[cv_idx]
y_test = y[cv_idx]
X_train = train2[~cv_idx]
y_train = y[~cv_idx]
return X_train, y_train, X_test, y_test<split>
test.loc[test['id'] == 3033,'budget'] = 250
test.loc[test['id'] == 3051,'budget'] = 50
test.loc[test['id'] == 3084,'budget'] = 337
test.loc[test['id'] == 3224,'budget'] = 4
test.loc[test['id'] == 3594,'budget'] = 25
test.loc[test['id'] == 3619,'budget'] = 500
test.loc[test['id'] == 3831,'budget'] = 3
test.loc[test['id'] == 3935,'budget'] = 500
test.loc[test['id'] == 4049,'budget'] = 995946
test.loc[test['id'] == 4424,'budget'] = 3
test.loc[test['id'] == 4460,'budget'] = 8
test.loc[test['id'] == 4555,'budget'] = 1200000
test.loc[test['id'] == 4624,'budget'] = 30
test.loc[test['id'] == 4645,'budget'] = 500
test.loc[test['id'] == 4709,'budget'] = 450
test.loc[test['id'] == 4839,'budget'] = 7
test.loc[test['id'] == 3125,'budget'] = 25
test.loc[test['id'] == 3142,'budget'] = 1
test.loc[test['id'] == 3201,'budget'] = 450
test.loc[test['id'] == 3222,'budget'] = 6
test.loc[test['id'] == 3545,'budget'] = 38
test.loc[test['id'] == 3670,'budget'] = 18
test.loc[test['id'] == 3792,'budget'] = 19
test.loc[test['id'] == 3881,'budget'] = 7
test.loc[test['id'] == 3969,'budget'] = 400
test.loc[test['id'] == 4196,'budget'] = 6
test.loc[test['id'] == 4221,'budget'] = 11
test.loc[test['id'] == 4222,'budget'] = 500
test.loc[test['id'] == 4285,'budget'] = 11
test.loc[test['id'] == 4319,'budget'] = 1
test.loc[test['id'] == 4639,'budget'] = 10
test.loc[test['id'] == 4719,'budget'] = 45
test.loc[test['id'] == 4822,'budget'] = 22
test.loc[test['id'] == 4829,'budget'] = 20
test.loc[test['id'] == 4969,'budget'] = 20
test.loc[test['id'] == 5021,'budget'] = 40
test.loc[test['id'] == 5035,'budget'] = 1
test.loc[test['id'] == 5063,'budget'] = 14
test.loc[test['id'] == 5119,'budget'] = 2
test.loc[test['id'] == 5214,'budget'] = 30
test.loc[test['id'] == 5221,'budget'] = 50
test.loc[test['id'] == 4903,'budget'] = 15
test.loc[test['id'] == 4983,'budget'] = 3
test.loc[test['id'] == 5102,'budget'] = 28
test.loc[test['id'] == 5217,'budget'] = 75
test.loc[test['id'] == 5224,'budget'] = 3
test.loc[test['id'] == 5469,'budget'] = 20
test.loc[test['id'] == 5840,'budget'] = 1
test.loc[test['id'] == 5960,'budget'] = 30
test.loc[test['id'] == 6506,'budget'] = 11
test.loc[test['id'] == 6553,'budget'] = 280
test.loc[test['id'] == 6561,'budget'] = 7
test.loc[test['id'] == 6582,'budget'] = 218
test.loc[test['id'] == 6638,'budget'] = 5
test.loc[test['id'] == 6749,'budget'] = 8
test.loc[test['id'] == 6759,'budget'] = 50
test.loc[test['id'] == 6856,'budget'] = 10
test.loc[test['id'] == 6858,'budget'] = 100
test.loc[test['id'] == 6876,'budget'] = 250
test.loc[test['id'] == 6972,'budget'] = 1
test.loc[test['id'] == 7079,'budget'] = 8000000
test.loc[test['id'] == 7150,'budget'] = 118
test.loc[test['id'] == 6506,'budget'] = 118
test.loc[test['id'] == 7225,'budget'] = 6
test.loc[test['id'] == 7231,'budget'] = 85
test.loc[test['id'] == 5222,'budget'] = 5
test.loc[test['id'] == 5322,'budget'] = 90
test.loc[test['id'] == 5350,'budget'] = 70
test.loc[test['id'] == 5378,'budget'] = 10
test.loc[test['id'] == 5545,'budget'] = 80
test.loc[test['id'] == 5810,'budget'] = 8
test.loc[test['id'] == 5926,'budget'] = 300
test.loc[test['id'] == 5927,'budget'] = 4
test.loc[test['id'] == 5986,'budget'] = 1
test.loc[test['id'] == 6053,'budget'] = 20
test.loc[test['id'] == 6104,'budget'] = 1
test.loc[test['id'] == 6130,'budget'] = 30
test.loc[test['id'] == 6301,'budget'] = 150
test.loc[test['id'] == 6276,'budget'] = 100
test.loc[test['id'] == 6473,'budget'] = 100
test.loc[test['id'] == 6842,'budget'] = 30
X = train.query('parentesco1==1')
 y = X['Target'] - 1
X = X.drop(['Target'], axis=1)
 np.random.seed(seed=1977)
 train2 = X.copy()
train_hhs = train2.idhogar
households = train2.idhogar.unique()
cv_hhs = np.random.choice(households, size=int(len(households)* 0.1), replace=False)
 cv_idx = np.isin(train2.idhogar, cv_hhs)
 X_test = train2[cv_idx]
y_test = y[cv_idx]
X_train = train2[~cv_idx]
y_train = y[~cv_idx]
X_train = train2
y_train = y
train_households = X_train.idhogar<define_variables>
trainAdditionalFeatures = pd.read_csv('.. /input/tmdb-competition-additional-features/TrainAdditionalFeatures.csv')
 testAdditionalFeatures = pd.read_csv('.. /input/tmdb-competition-additional-features/TestAdditionalFeatures.csv')
 train = pd.merge(train, trainAdditionalFeatures, how='left', on=['imdb_id'])
 test = pd.merge(test, testAdditionalFeatures, how='left', on=['imdb_id'] )
extra_drop_features = ['geo_manual_elec_LE_2',
'agg18_estadocivil5_COUNT',
'agg18_estadocivil7_COUNT',
'geo_pared_LE_5',
'agg18_estadocivil6_COUNT',
'geo_pared_LE_4',
'agg18_parentesco10_COUNT',
'num_over_18',
'agg18_estadocivil1_MEAN',
'agg18_estadocivil4_COUNT',
'r4t3_to_tamhog',
'agg18_estadocivil3_COUNT',
'geo_pared_LE_3',
'geo_pared_LE_6',
'geo_pared_LE_0',
'geo_pared_LE_7',
'agg18_parentesco7_COUNT',
'agg18_parentesco10_MEAN',
'agg18_parentesco8_MEAN',
'parentesco1',
'rez_esc',
'dependency_yes',
'hacdor',
'hacapo',
'fe_people_weird_stat',
'geo_hogar_mayor',
'geo_eviv_LE_0',
'geo_epared_LE_0',
'geo_epared_LE_2',
'geo_elimbasu_LE_4',
'geo_energcocinar_LE_0',
'agg18_parentesco9_COUNT',
'agg18_parentesco8_COUNT',
'agg18_parentesco11_COUNT',
'parentesco_LE',
'agg18_parentesco6_MEAN',
'agg18_parentesco6_COUNT',
'agg18_parentesco5_COUNT',
'agg18_parentesco4_MEAN',
'agg18_parentesco4_COUNT',
'geo_energcocinar_LE_1',
'agg18_parentesco3_COUNT',
'geo_energcocinar_LE_2',
'agg18_parentesco2_COUNT',
'agg18_parentesco1_COUNT',
'agg18_parentesco12_MEAN',
'agg18_parentesco12_COUNT',
'dependency_no'] + ['geo_hogar_nin', 'geo_manual_elec_LE_3']<drop_column>
print("Missing rating in Train set", train['rating'].isna().sum())
 print("Missing total Votes in Train set", train['totalVotes'].isna().sum())
 print("")
 print("Missing rating in Test set", test['rating'].isna().sum())
 print("Missing total Votes in Test set", test['totalVotes'].isna().sum() )
lgb_drop_cols = extra_drop_features + ["idhogar"]
try:
X_train.drop(lgb_drop_cols, axis=1, inplace=True)
 X_test.drop(lgb_drop_cols, axis=1, inplace=True)
 except:
print("Error dropping columns" )<compute_train_metric>
data.drop(['imdb_id','poster_path'],axis=1,inplace=True)
 test.drop(['imdb_id','poster_path'],axis=1,inplace=True )
opt_parameters = {'colsample_bytree': 0.88, 'min_child_samples': 90, 'num_leaves': 16, 'subsample': 0.94}
def evaluate_macroF1_lgb(truth, predictions):
pred_labels = predictions.reshape(len(np.unique(truth)) ,-1 ).argmax(axis=0)
 f1 = f1_score(truth, pred_labels, average='macro')
 return('macroF1', f1, True)
 fit_params={"early_stopping_rounds":300,
"eval_metric" : evaluate_macroF1_lgb,
"eval_set" : [(X_train,y_train),(X_test,y_test)],
'eval_names': ['train', 'valid'],
'verbose': False,
'categorical_feature': 'auto'}
def learning_rate_power_0997(current_iter):
base_learning_rate = 0.1
min_learning_rate = 0.02
lr = base_learning_rate * np.power (.995, current_iter)
 return max(lr, min_learning_rate)
 fit_params['verbose'] = 200
fit_params['callbacks'] = [lgb.reset_parameter(learning_rate=learning_rate_power_0997)]<train_model>
data.isnull().sum()
def _parallel_fit_estimator(estimator1, X, y, sample_weight=None, threshold=True, **fit_params):
estimator = clone(estimator1)
 X_train, y_train, X_test, y_test = split_data(X, y, households=train_households, seed=None)
 fit_params["eval_set"] = [(X_train,y_train),(X_test,y_test)]
if sample_weight is not None:
if isinstance(estimator1, ExtraTreesClassifier)or isinstance(estimator1, RandomForestClassifier):
estimator.fit(X_train, y_train)
 else:
estimator.fit(X_train, y_train, sample_weight=sample_weight, **fit_params)
 else:
if isinstance(estimator1, ExtraTreesClassifier)or isinstance(estimator1, RandomForestClassifier):
estimator.fit(X_train, y_train)
 else:
estimator.fit(X_train, y_train, **fit_params)
 if not isinstance(estimator1, ExtraTreesClassifier)and not isinstance(estimator1, RandomForestClassifier):
best_cv_round = np.argmax(estimator.evals_result_['valid']['macroF1'])
 best_cv = np.max(estimator.evals_result_['valid']['macroF1'])
 best_train = estimator.evals_result_['train']['macroF1'][best_cv_round]
else:
best_train = f1_score(y_train, estimator.predict(X_train), average="macro")
 best_cv = f1_score(y_test, estimator.predict(X_test), average="macro")
 print("Train F1:", best_train)
 print("Test F1:", best_cv)
 if threshold:
if(( best_cv > 0.39)and(best_train > 0.75)) or(( best_cv > 0.44)and(best_train > 0.65)) :
return estimator
else:
print("Unacceptable!!! Trying again...")
 return _parallel_fit_estimator(estimator1, X, y, sample_weight=sample_weight, **fit_params)
 else:
return estimator
class VotingClassifierLGBM(VotingClassifier):

 def fit(self, X, y, sample_weight=None, threshold=True, **fit_params):
if isinstance(y, np.ndarray)and len(y.shape)> 1 and y.shape[1] > 1:
raise NotImplementedError('Multilabel and multi-output'
' classification is not supported.')
 if self.voting not in('soft', 'hard'):
raise ValueError("Voting must be 'soft' or 'hard'; got(voting=%r)"
% self.voting)
 if self.estimators is None or len(self.estimators)== 0:
raise AttributeError('Invalid `estimators` attribute, `estimators`'
' should be a list of(string, estimator)'
' tuples')
 if(self.weights is not None and
len(self.weights)!= len(self.estimators)) :
raise ValueError('Number of classifiers and weights must be equal'
'; got %d weights, %d estimators'
%(len(self.weights), len(self.estimators)))
 if sample_weight is not None:
for name, step in self.estimators:
if not has_fit_parameter(step, 'sample_weight'):
raise ValueError('Underlying estimator '%s' does not'
' support sample weights.' % name)
 names, clfs = zip(*self.estimators)
 self._validate_names(names)
 n_isnone = np.sum([clf is None for _, clf in self.estimators])
 if n_isnone == len(self.estimators):
raise ValueError('All estimators are None.At least one is '
'required to be a classifier!')
 self.le_ = LabelEncoder().fit(y)
 self.classes_ = self.le_.classes_
self.estimators_ = []
transformed_y = self.le_.transform(y)
 self.estimators_ = Parallel(n_jobs=self.n_jobs )(
 delayed(_parallel_fit_estimator )(clone(clf), X, transformed_y,
sample_weight=sample_weight, threshold=threshold, **fit_params)
 for clf in clfs if clf is not None)
 return self<train_model>
data.isnull().sum()
clfs = []
for i in range(24):
clf = lgb.LGBMClassifier(max_depth=-1, learning_rate=0.1, objective='multiclass',
random_state=314+i, silent=True, metric='None',
n_jobs=4, n_estimators=5000, class_weight='balanced')
 clf.set_params(**opt_parameters)
 clfs.append(( 'lgbm{}'.format(i), clf))
vc = VotingClassifierLGBM(clfs, voting='soft')
 del clfs
_ = vc.fit(X_train, y_train, **fit_params)
 clf_final = vc.estimators_[0]<compute_train_metric>
data=data.drop(['homepage'],axis =1)
 test=test.drop(['homepage'],axis =1 )
global_score = f1_score(y_test, clf_final.predict(X_test), average='macro')
 vc.voting = 'soft'
global_score_soft = f1_score(y_test, vc.predict(X_test), average='macro')
 vc.voting = 'hard'
global_score_hard = f1_score(y_test, vc.predict(X_test), average='macro')
 print('Validation score of a single LGBM Classifier: {:.4f}'.format(global_score))
print('Validation score of a VotingClassifier on 3 LGBMs with soft voting strategy: {:.4f}'.format(global_score_soft))
print('Validation score of a VotingClassifier on 3 LGBMs with hard voting strategy: {:.4f}'.format(global_score_hard))<compute_train_metric>
data=data.drop(['belongs_to_collection'],axis =1)
 test=test.drop(['belongs_to_collection'],axis =1 )
global_score = f1_score(y_test, clf_final.predict(X_test), average='macro')
 vc.voting = 'soft'
global_score_soft = f1_score(y_test, vc.predict(X_test), average='macro')
 vc.voting = 'hard'
global_score_hard = f1_score(y_test, vc.predict(X_test), average='macro')
 print('Validation score of a single LGBM Classifier: {:.4f}'.format(global_score))
print('Validation score of a VotingClassifier on 3 LGBMs with soft voting strategy: {:.4f}'.format(global_score_soft))
print('Validation score of a VotingClassifier on 3 LGBMs with hard voting strategy: {:.4f}'.format(global_score_hard))<compute_test_metric>
genres = {}
for i in data['genres']:
if(not(pd.isnull(i))):
if(eval(i)[0]['name'])not in genres:
genres[eval(i)[0]['name']]=1
else:
genres[eval(i)[0]['name']]+=1
ranked_features = feature_importance(clf_final, X_train )<create_dataframe>
genres_count=[]
for i in data['genres']:
if(not(pd.isnull(i))):
genres_count.append(len(eval(i)))
 else:
genres_count.append(0)
 data['num_genres'] = genres_count
y_subm = pd.DataFrame()
y_subm['Id'] = test_ids<predict_on_test>
genres_count_test=[]
for i in test['genres']:
if(not(pd.isnull(i))):
genres_count_test.append(len(eval(i)))
 else:
genres_count_test.append(0)
 test['num_genres'] = genres_count_test
y_subm['Target'] = clf_final.predict(test[X_train.columns])+ 1
vc.voting = 'soft'
y_subm_soft = y_subm.copy(deep=True)
 y_subm_soft['Target'] = vc.predict(test[X_train.columns])+ 1
vc.voting = 'hard'
y_subm_hard = y_subm.copy(deep=True)
 y_subm_hard['Target'] = vc.predict(test[X_train.columns])+ 1<save_to_csv>
data.drop(['genres'],axis=1, inplace = True)
 test.drop(['genres'],axis=1, inplace = True )
now = datetime.now()
sub_file = 'submission_LGB_{:.4f}_{}.csv'.format(global_score, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_soft = 'submission_soft_LGB_{:.4f}_{}.csv'.format(global_score_soft, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_hard = 'submission_hard_LGB_{:.4f}_{}.csv'.format(global_score_hard, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_0forNonHeads = 'submission_0forNonHead_LGB_{:.4f}_{}.csv'.format(global_score, str(now.strftime('%Y-%m-%d-%H-%M')))
 y_subm.to_csv(sub_file, index=False)
 y_subm_soft.to_csv(sub_file_soft, index=False)
 y_subm_hard.to_csv(sub_file_hard, index=False )<import_modules>
prod_comp_count=[]
for i in data['production_companies']:
if(not(pd.isnull(i))):
prod_comp_count.append(len(eval(i)))
 else:
prod_comp_count.append(0)
 data['num_prod_companies'] = prod_comp_count
%matplotlib inline<load_from_csv>
prod_comp_count_test=[]
for i in test['production_companies']:
if(not(pd.isnull(i))):
prod_comp_count_test.append(len(eval(i)))
 else:
prod_comp_count_test.append(0)
 test['num_prod_companies'] = prod_comp_count_test
def read_data() :
print("
train = pd.read_csv(".. /input/train.csv", index_col = 0)
 test = pd.read_csv(".. /input/test.csv", index_col = 0)
 sub = pd.read_csv(".. /input/sample_submission.csv", index_col = 0)
 print("
Train shape: {}
Test shape: {}".format(train.shape, test.shape))
return train, test, sub<split>
data.drop(['production_companies'],axis=1, inplace = True)
 test.drop(['production_companies'],axis=1, inplace = True )
train_orig, test_orig, sub_orig = read_data()<prepare_x_and_y>
prod_coun_count=[]
for i in data['production_countries']:
if(not(pd.isnull(i))):
prod_coun_count.append(len(eval(i)))
 else:
prod_coun_count.append(0)
 data['num_prod_countries'] = prod_coun_count
y_train = train_orig['Target']
x_train = train_orig.drop('Target', 1)
 x_test = test_orig.copy()
test_id = test_orig.index<prepare_x_and_y>
prod_coun_count_test=[]
for i in test['production_countries']:
if(not(pd.isnull(i))):
prod_coun_count_test.append(len(eval(i)))
 else:
prod_coun_count_test.append(0)
 test['num_prod_countries'] = prod_coun_count_test
class MissingValuesImputer(BaseEstimator, TransformerMixin):
def __init__(self, impute_zero_columns):
self.impute_zero_columns = impute_zero_columns
def fit(self, X, y = None):
print("Mean Values Imputer")
 return self
def transform(self, X, y = None):
for column in self.impute_zero_columns:
X[column] = X[column].fillna(0)
 self.X_with_meaneduc_na = X[pd.isnull(X['meaneduc'])]
self.mean_escolari_dict = dict(self.X_with_meaneduc_na.groupby('idhogar')['escolari'].apply(np.mean))
for row_index in self.X_with_meaneduc_na.index:
row_idhogar = X.at[row_index, 'idhogar']
X.at[row_index, 'meaneduc'] = self.mean_escolari_dict[row_idhogar]
X.at[row_index, 'SQBmeaned'] = np.square(self.mean_escolari_dict[row_idhogar])
 return X<prepare_x_and_y>
data.drop(['production_countries'],axis=1, inplace = True)
 test.drop(['production_countries'],axis=1, inplace = True )
class CategoricalVariableTransformer(BaseEstimator, TransformerMixin):
def __init__(self):
self.categorical_columns = ['idhogar']
def fit(self, X, y = None):
print("Categorical Variables Transformer")
 return self
def transform(self, X, y = None):
X['dependency'] = np.sqrt(X['SQBdependency'])
 X.loc[X['edjefe'] == 'no', 'edjefe'] = '0'
X.loc[X['edjefe'] == 'yes', 'edjefe'] = '1'
X.loc[X['edjefa'] == 'no', 'edjefa'] = '0'
X.loc[X['edjefa'] == 'yes', 'edjefa'] = '1'
X['edjefa'] = X['edjefa'].astype(int)
 X['edjefe'] = X['edjefe'].astype(int)
 label_encoder = LabelEncoder()
for column in self.categorical_columns:
X[column] = label_encoder.fit_transform(X[column])
 return X<normalization>
total_cast=[]
for i in data['cast']:
if(not(pd.isnull(i))):
total_cast.append(len(eval(i)))
 else:
total_cast.append(0)
 data['cast_count'] = total_cast
class UnnecessaryColumnsRemoverTransformer(BaseEstimator, TransformerMixin):
def __init__(self, axis = 1):
print("Unnecessary Columns Remover Transformer")
 self.axis = axis
self.unnecessary_columns = [
'r4t3', 'tamhog', 'tamviv', 'hogar_total', 'v18q', 'v14a', 'agesq',
'mobilephone', 'energcocinar1', 'sanitario6',
'estadocivil7', 'lugar1', 'area1', 'female'
]
def fit(self, X, y = None):
unnecessary_columns_to_extend = [
[col for col in X.columns.tolist() if 'SQB' in col],
[col for col in X.columns.tolist() if 'epared' in col],
[col for col in X.columns.tolist() if 'etecho' in col],
[col for col in X.columns.tolist() if 'eviv' in col],
[col for col in X.columns.tolist() if 'instlevel' in col],
[col for col in X.columns.tolist() if 'pared' in col],
[col for col in X.columns.tolist() if 'piso' in col],
[col for col in X.columns.tolist() if 'techo' in col],
[col for col in X.columns.tolist() if 'abastagua' in col],
[col for col in X.columns.tolist() if 'elimbasu' in col],
[col for col in X.columns.tolist() if 'tipoviv' in col]
]
for col_list in unnecessary_columns_to_extend:
self.unnecessary_columns.extend(col_list)
 return self
def transform(self, X, y = None):
X = X.drop(self.unnecessary_columns, axis = self.axis)
 return X<train_model>
total_cast=[]
for i in test['cast']:
if(not(pd.isnull(i))):
total_cast.append(len(eval(i)))
 else:
total_cast.append(0)
 test['cast_count'] = total_cast
class ReverseOHETransformer(BaseEstimator, TransformerMixin):
def __init__(self, axis = 1):
self.axis = axis
self.ohe_column_prefixes = ['epared', 'etecho', 'eviv', 'instlevel',
'pared', 'piso', 'techo', 'abastagua']
def fit(self, X, y = None):
print("Reverse OHE Transformer")
 self.ohe_columns_dict = dict()
for column_prefix in self.ohe_column_prefixes:
ohe_columns = [col for col in X.columns if col.startswith(column_prefix)]
self.ohe_columns_dict[column_prefix] = ohe_columns
return self
def transform(self, X, y = None):
for ohe_column_prefix, ohe_column_list in self.ohe_columns_dict.items() :
ohe_df = X[ohe_column_list]
ohe_df.columns = list(range(ohe_df.shape[1]))
ohe_numeric_df = ohe_df.idxmax(axis = self.axis)
 ohe_numeric_df.name = ohe_column_prefix
X = pd.concat([X, ohe_numeric_df], axis = self.axis)
 X = X.drop(ohe_column_list, axis = self.axis)
 return X<feature_engineering>
data= data.drop(['cast'],axis=1)
 test= test.drop(['cast'],axis=1 )
class FeatureEngineeringTransformer(BaseEstimator, TransformerMixin):
def __init__(self, axis = 1):
self.axis = axis
self.individual_boolean_features = ['dis', 'male', 'estadocivil1', 'estadocivil2',
'estadocivil3', 'estadocivil4', 'estadocivil5', 'estadocivil6',
'parentesco1', 'parentesco2', 'parentesco3', 'parentesco4',
'parentesco5', 'parentesco6', 'parentesco7', 'parentesco8',
'parentesco9', 'parentesco10', 'parentesco11']
self.individual_ordered_features = ['escolari', 'age']
def fit(self, X, y = None):
print("Feature Engineering Transformer")
 self.more_columns_to_drop = [
[col for col in X.columns.tolist() if 'parentesco' in col and 'parentesco1' not in col],
['idhogar']
]
f = lambda x: x.std(ddof = 0)
 f.__name__ = 'std_0'
self.aggregate_features =(['mean', 'max', 'min', 'sum', f])
 return self
def transform(self, X, y = None):
X['rent_per_room'] = X['v2a1']/X['rooms']
X['adults_per_room'] = X['hogar_adul']/X['rooms']
X['males_per_room'] = X['r4h3']/X['rooms']
X['females_per_room'] = X['r4m3']/X['rooms']
X['children_per_room'] = X['hogar_nin']/X['rooms']
X['humans_per_room'] = X['hhsize']/X['rooms']
X['beds_per_room'] = X['bedrooms']/X['rooms']
X['adults_per_bedroom'] = X['hogar_adul']/X['bedrooms']
X['males_per_bedroom'] = X['r4h3']/X['bedrooms']
X['females_per_bedroom'] = X['r4m3']/X['bedrooms']
X['children_per_bedroom'] = X['hogar_nin']/X['bedrooms']
X['humans_per_bedroom'] = X['hhsize']/X['bedrooms']
X['persons12less_fraction'] =(X['r4h1'] + X['r4m1'])/X['hhsize']
X['males12plus_fraction'] = X['r4h2']/X['hhsize']
X['total_males_fraction'] = X['r4h3']/X['hhsize']
X['females12plus_fraction'] = X['r4m2']/X['hhsize']
X['all_females_fraction'] = X['r4m3']/X['hhsize']
X['rent_per_person'] = X['v2a1']/X['hhsize']
X['mobiles_per_person'] = X['qmobilephone']/X['hhsize']
X['tablets_per_person'] = X['v18q1']/X['hhsize']
X['mobiles_per_male'] = X['qmobilephone']/X['r4h3']
X['tablets_per_male'] = X['v18q1']/X['r4h3']
grouped_df = X.groupby('idhogar')[self.individual_boolean_features + self.individual_ordered_features]
grouped_df = grouped_df.agg(self.aggregate_features)
 X = X.join(grouped_df, on = 'idhogar')
 for col in self.more_columns_to_drop:
X = X.drop(col, axis = self.axis)
 return X<train_on_grid>
total_crew=[]
for i in data['crew']:
if(not(pd.isnull(i))):
total_crew.append(len(eval(i)))
 else:
total_crew.append(0)
 data['crew_count'] = total_crew
class LGBClassifierCV(BaseEstimator, RegressorMixin):
def __init__(self, axis = 0, lgb_params = None, fit_params = None, cv = 3, perform_bayes_search = False, perform_random_search = False, use_train_test_split = False, use_kfold_split = True):
self.axis = axis
self.lgb_params = lgb_params
self.fit_params = fit_params
self.cv = cv
self.perform_random_search = perform_random_search
self.perform_bayes_search = perform_bayes_search
self.use_train_test_split = use_train_test_split
self.use_kfold_split = use_kfold_split
@property
def feature_importances_(self):
feature_importances = []
for estimator in self.estimators_:
feature_importances.append(
 estimator.feature_importances_ 
)
 return np.mean(feature_importances, axis = 0)
 @property
def evals_result_(self):
evals_result = []
for estimator in self.estimators_:
evals_result.append(
 estimator.evals_result_ 
)
 return np.array(evals_result)
 @property
def best_scores_(self):
best_scores = []
for estimator in self.estimators_:
best_scores.append(
 estimator.best_score_['validation']['macroF1'] 
)
 return np.array(best_scores)
 @property
def cv_scores_(self):
return self.best_scores_
@property
def cv_score_(self):
return np.mean(self.best_scores_)
 @property
def best_iterations_(self):
best_iterations = []
for estimator in self.estimators_:
best_iterations.append(
 estimator.best_iteration_ 
)
 return np.array(best_iterations)
 @property
def best_iteration_(self):
return np.round(np.mean(self.best_iterations_))
def find_best_params_(self, X, y):
if self.perform_random_search:
lgb_search_params = {
'num_leaves': sp_randint(20, 100),
'min_child_samples': sp_randint(40, 100),
'min_child_weight': [1e-5, 1e-3, 1e-2, 1e-1, 1, 1e1, 1e2, 1e3, 1e4],
'subsample': sp_uniform(loc = 0.75, scale = 0.25),
'colsample_bytree': sp_uniform(loc = 0.8, scale = 0.15),
'reg_alpha': [0, 1e-3, 1e-1, 1, 10, 50, 100],
'reg_lambda': [0, 1e-3, 1e-1, 1, 10, 50, 100]
}
x_train, x_val, y_train, y_val = train_test_split(X, y, test_size = 0.10, random_state = 42, stratify = y)
 F1_scorer = make_scorer(f1_score, greater_is_better = True, average = 'macro')
 lgb_model = lgb.LGBMClassifier(**self.lgb_params)
 self.fit_params["eval_set"] = [(x_train, y_train),(x_val, y_val)]
self.fit_params["verbose"] = 200
rs = RandomizedSearchCV(estimator = lgb_model,
param_distributions = lgb_search_params,
n_iter = 100,
scoring = F1_scorer,
cv = 5,
refit = True,
random_state = 314,
verbose = False,
fit_params = self.fit_params)
 _ = rs.fit(x_train, y_train)
 optimal_params = rs.best_params_
if self.perform_bayes_search:
init_round = 10
opt_roun = 10
n_folds = 6
random_seed = 42
n_estimators = 500
learning_rate = 0.02
colsample_bytree = 0.93
train_data = lgb.Dataset(data = X, label = y)
 def lgb_eval(num_leaves, feature_fraction, bagging_fraction, max_depth, lambda_l1, lambda_l2, min_split_gain, min_child_weight, colsample_bytree, min_child_samples, subsample):
params = {'application': 'multiclass',
'num_iterations': n_estimators,
'learning_rate': learning_rate,
'early_stopping_round': 300,
'metric': 'macroF1'}
params["num_leaves"] = int(round(num_leaves))
params["num_class"] = 5
params['feature_fraction'] = max(min(feature_fraction, 1), 0)
 params['bagging_fraction'] = max(min(bagging_fraction, 1), 0)
 params['max_depth'] = int(round(max_depth))
params['lambda_l1'] = max(lambda_l1, 0)
 params['lambda_l2'] = max(lambda_l2, 0)
 params['min_split_gain'] = min_split_gain
params['min_child_weight'] = min_child_weight
params['colsample_bytree'] = 0.93
params['min_child_samples'] = 56,
params['subsample'] = 0.84
cv_result = lgb.cv(params, train_data, nfold = n_folds, seed = random_seed, stratified = True, verbose_eval = 200, metrics = ['auc'])
 return max(cv_result['auc-mean'])
 lgbBO = BayesianOptimization(lgb_eval, {'num_leaves':(19, 45),
'feature_fraction':(0.1, 0.9),
'bagging_fraction':(0.8, 1),
'max_depth':(5, 8.99),
'lambda_l1':(0, 5),
'lambda_l2':(0, 3),
'min_split_gain':(0.001, 0.1),
'min_child_weight':(5, 50),
'colsample_bytree' :(0.7,1.0),
'min_child_samples' :(40,65),
'subsample' :(0.7,1.0)
 }, random_state = 0)
 lgbBO.maximize(init_points = init_round, n_iter = opt_roun)
 optimal_params = lgbBO.res['max']['max_params']
print("Optimal LGB parameters:")
 print(optimal_params)
 with open("lgb_best_params.pickle", "wb")as lgb_best_params:
pickle.dump(optimal_params, lgb_best_params)
 return optimal_params
def fit(self, X, y, **fit_params):
print("LGBClassifierCV")
 X.insert(0, 'Target', y)
 X = X.query('parentesco1 == 1')
 y = X['Target'] - 1
X = X.drop(['Target', 'parentesco1'], 1)
 print("Number of columns in train - " + str(X.shape[1]))
self.estimators_ = []
if self.perform_bayes_search or self.perform_random_search:
self.lgb_optimal_params = {'num_leaves': 19,
'feature_fraction': 0.10676970062446138,
'bagging_fraction': 0.8,
'max_depth': 9,
'lambda_l1': 5.0,
'lambda_l2': 2.999999999999948,
'min_split_gain': 0.1,
'min_child_weight': 49.999999999999986,
'colsample_bytree': 0.7,
'min_child_samples': 65,
'subsample': 1.0}
if self.use_train_test_split:
x_train, x_val, y_train, y_val = train_test_split(X, y, test_size = 0.1, random_state = 0)
 lgb_model = lgb.LGBMClassifier(**self.lgb_params)
 if self.perform_random_search or self.perform_bayes_search:
lgb_model.set_params(**self.lgb_optimal_params)
 lgb_model.fit(
 x_train, y_train,
eval_set = [(x_train, y_train),(x_val, y_val)],
**self.fit_params 
)
 print("Train F1 - " + str(lgb_model.best_score_['train']['macroF1'])+ " " + "Validation F1 - " + str(lgb_model.best_score_['validation']['macroF1']))
self.estimators_.append(lgb_model)
 if self.use_kfold_split:
kf = StratifiedKFold(n_splits = self.cv, shuffle = True)
 for fold_index,(train, valid)in enumerate(kf.split(X, y)) :
print("Train Fold Index - " + str(fold_index))
lgb_model = lgb.LGBMClassifier(**self.lgb_params)
 if self.perform_random_search:
lgb_model.set_params(**self.lgb_optimal_params)
 lgb_model.fit(
 X.iloc[train], y.iloc[train],
eval_set = [(X.iloc[train], y.iloc[train]),(X.iloc[valid], y.iloc[valid])],
**self.fit_params 
)
 print("Train F1 - " + str(lgb_model.best_score_['train']['macroF1'])+ " " + "Validation F1 - " + str(lgb_model.best_score_['validation']['macroF1']))
self.estimators_.append(lgb_model)
 return self
def predict(self, X):
X = X.drop('parentesco1', 1)
 y_pred = []
for estimator_index, estimator in enumerate(self.estimators_):
print("Estimator Index - " + str(estimator_index))
y_pred.append(estimator.predict(X))
return np.mean(y_pred, axis = self.axis ).astype(int )<compute_train_metric>
total_crew=[]
for i in test['crew']:
if(not(pd.isnull(i))):
total_crew.append(len(eval(i)))
 else:
total_crew.append(0)
 test['crew_count'] = total_crew
def get_lgb_params() :
def evaluate_macroF1_lgb(truth, predictions):
pred_labels = predictions.reshape(len(np.unique(truth)) , -1 ).argmax(axis = 0)
 f1 = f1_score(truth, pred_labels, average = 'macro')
 return('macroF1', f1, True)
 def learning_rate_power_0997(current_iter):
base_learning_rate = 0.1
min_learning_rate = 0.02
lr = base_learning_rate * np.power (.995, current_iter)
 return max(lr, min_learning_rate)
 lgb_params = {'boosting_type': 'dart',
'class_weight': 'balanced',
"objective": 'multiclassova',
'metric': None,
'silent': True,
'random_state': 0,
'n_jobs': -1}
fit_params={"early_stopping_rounds": 400,
"eval_metric" : evaluate_macroF1_lgb,
'eval_names': ['train', 'validation'],
'verbose': False,
'categorical_feature': 'auto'}
return lgb_params, fit_params<feature_engineering>
data= data.drop(['crew'],axis=1)
 test= test.drop(['crew'],axis=1 )
lgb_params, lgb_fit_params = get_lgb_params()
pipeline = Pipeline([ 
('na_imputer', MissingValuesImputer(impute_zero_columns = ['v18q1', 'v2a1', 'rez_esc'])) , 
('cat_transformer', CategoricalVariableTransformer()), 
('unnecessary_columns_remover_transformer', UnnecessaryColumnsRemoverTransformer()), 
('feature_engineering_transformer', FeatureEngineeringTransformer()), 
('lgb', LGBClassifierCV(lgb_params = lgb_params,
fit_params = lgb_fit_params,
cv = 5,
perform_random_search = False,
perform_bayes_search = True,
use_train_test_split = True,
use_kfold_split = False)
)
 ])
 pipeline.fit(x_train.copy() , y_train.copy())
 pred = pipeline.predict(x_test.copy())
 print("Local CV Score - " + str(pipeline.named_steps['lgb'].cv_score_))
sub_orig['Target'] = pred + 1
sub_orig.to_csv('Pipeline_Base_LGB_'+ str(pipeline.named_steps['lgb'].cv_score_)+ '.csv')
 print(sub_orig.head() )<set_options>
data= data.drop(['original_title'],axis=1)
 test= test.drop(['original_title'],axis=1 )
%matplotlib inline
plt.style.use('fivethirtyeight')
 plt.rcParams['font.size'] = 18
plt.rcParams['patch.edgecolor'] = 'k'<load_from_csv>
data['original_language'] =data['original_language'].apply(lambda x: 1 if x=='en' else(2 if x=='zh' else 0))
test['original_language'] =test['original_language'].apply(lambda x: 1 if x=='en' else(2 if x=='zh' else 0))
pd.options.display.max_columns = 150
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.head()<data_type_conversions>
data['release_date']=pd.to_datetime(data['release_date'])
 test['release_date']=pd.to_datetime(data['release_date'] )
mapping = {"yes": 1, "no": 0}
for df in [train, test]:
df['dependency'] = df['dependency'].replace(mapping ).astype(np.float64)
 df['edjefa'] = df['edjefa'].replace(mapping ).astype(np.float64)
 df['edjefe'] = df['edjefe'].replace(mapping ).astype(np.float64)
 train[['dependency', 'edjefa', 'edjefe']].describe()<concatenate>
data['release_day']=data['release_date'].dt.dayofweek
test['release_day']=test['release_date'].dt.dayofweek
test['Target'] = np.nan
data = train.append(test, ignore_index = True )<count_unique_values>
test['release_day']=test['release_day'].fillna(0 )
all_equal = train.groupby('idhogar')['Target'].apply(lambda x: x.nunique() == 1)
 not_equal = all_equal[all_equal != True]
print('There are {} households where the family members do not all have the same target.'.format(len(not_equal)) )<filter>
data.drop(['release_date'],axis=1,inplace=True)
 test.drop(['release_date'],axis=1,inplace=True )
train[train['idhogar'] == not_equal.index[0]][['idhogar', 'parentesco1', 'Target']]<groupby>
print("train data")
 print(data['status'].value_counts())
 print("test data")
 test['status'].value_counts()
households_leader = train.groupby('idhogar')['parentesco1'].sum()
households_no_head = train.loc[train['idhogar'].isin(households_leader[households_leader == 0].index), :]
print('There are {} households without a head.'.format(households_no_head['idhogar'].nunique()))<groupby>
data.drop(['status'],axis=1,inplace =True)
 test.drop(['status'],axis=1,inplace =True )
households_no_head_equal = households_no_head.groupby('idhogar')['Target'].apply(lambda x: x.nunique() == 1)
 print('{} Households with no head have different labels.'.format(sum(households_no_head_equal == False)) )<feature_engineering>
Keywords_count=[]
for i in data['Keywords']:
if(not(pd.isnull(i))):
Keywords_count.append(len(eval(i)))
 else:
Keywords_count.append(0)
 data['Keywords_count'] = Keywords_count
for household in not_equal.index:
true_target = int(train[(train['idhogar'] == household)&(train['parentesco1'] == 1.0)]['Target'])
 train.loc[train['idhogar'] == household, 'Target'] = true_target
all_equal = train.groupby('idhogar')['Target'].apply(lambda x: x.nunique() == 1)
 not_equal = all_equal[all_equal != True]
print('There are {} households where the family members do not all have the same target.'.format(len(not_equal)) )<feature_engineering>
Keywords_count=[]
for i in test['Keywords']:
if(not(pd.isnull(i))):
Keywords_count.append(len(eval(i)))
 else:
Keywords_count.append(0)
 test['Keywords_count'] = Keywords_count
missing = pd.DataFrame(data.isnull().sum() ).rename(columns = {0: 'total'})
 missing['percent'] = missing['total'] / len(data)
 missing.sort_values('percent', ascending = False ).head(10 ).drop('Target' )<groupby>
data=data.drop(['Keywords'],axis=1)
 data=data.drop(['title'],axis=1)
 test=test.drop(['Keywords'],axis=1)
 test=test.drop(['title'],axis=1 )
heads.groupby('v18q')['v18q1'].apply(lambda x: x.isnull().sum() )<data_type_conversions>
data.drop(['tagline'],axis=1,inplace =True)
 test.drop(['tagline'],axis=1,inplace =True )
data['v18q1'] = data['v18q1'].fillna(0 )<feature_engineering>
data['runtime']=data['runtime'].fillna(data['runtime'].mean())
 test['runtime']=test['runtime'].fillna(test['runtime'].mean() )
data.loc[(data['tipovivi1'] == 1), 'v2a1'] = 0
data['v2a1-missing'] = data['v2a1'].isnull()
data['v2a1-missing'].value_counts()<feature_engineering>
spoken_count=[]
for i in data['spoken_languages']:
if(not(pd.isnull(i))):
spoken_count.append(len(eval(i)))
 else:
spoken_count.append(0)
 data['spoken_count'] = spoken_count
spoken_count_test=[]
for i in test['spoken_languages']:
if(not(pd.isnull(i))):
spoken_count_test.append(len(eval(i)))
 else:
spoken_count_test.append(0)
 test['spoken_count'] = spoken_count_test
data.loc[(( data['age'] > 19)|(data['age'] < 7)) &(data['rez_esc'].isnull()), 'rez_esc'] = 0
data['rez_esc-missing'] = data['rez_esc'].isnull()<feature_engineering>
data.drop(['spoken_languages'],axis=1,inplace=True)
 test.drop(['spoken_languages'],axis=1,inplace=True )
data.loc[data['rez_esc'] > 5, 'rez_esc'] = 5<define_variables>
data['budget'] = np.log1p(data['budget'])
 test['budget'] = np.log1p(test['budget'] )
id_ = ['Id', 'idhogar', 'Target']<define_variables>
y= data['revenue'].values
cols = [col for col in data.columns if col not in ['revenue', 'id']]
X= data[cols].values
y = np.log1p(y )
ind_bool = ['v18q', 'dis', 'male', 'female', 'estadocivil1', 'estadocivil2', 'estadocivil3',
'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco1', 'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5',
'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12', 'instlevel1', 'instlevel2', 'instlevel3',
'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8',
'instlevel9', 'mobilephone', 'rez_esc-missing']
ind_ordered = ['rez_esc', 'escolari', 'age']<define_variables>
clf = LinearRegression()
scores = cross_val_score(clf, X, y, scoring="neg_mean_squared_error", cv=10)
 rmse_scores = np.sqrt(-scores)
 print(rmse_scores.mean() )
hh_bool = ['hacdor', 'hacapo', 'v14a', 'refrig', 'paredblolad', 'paredzocalo',
'paredpreb','pisocemento', 'pareddes', 'paredmad',
'paredzinc', 'paredfibras', 'paredother', 'pisomoscer', 'pisoother',
'pisonatur', 'pisonotiene', 'pisomadera',
'techozinc', 'techoentrepiso', 'techocane', 'techootro', 'cielorazo',
'abastaguadentro', 'abastaguafuera', 'abastaguano',
'public', 'planpri', 'noelec', 'coopele', 'sanitario1',
'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6',
'energcocinar1', 'energcocinar2', 'energcocinar3', 'energcocinar4',
'elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4',
'elimbasu5', 'elimbasu6', 'epared1', 'epared2', 'epared3',
'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3',
'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5',
'computer', 'television', 'lugar1', 'lugar2', 'lugar3',
'lugar4', 'lugar5', 'lugar6', 'area1', 'area2', 'v2a1-missing']
hh_ordered = [ 'rooms', 'r4h1', 'r4h2', 'r4h3', 'r4m1','r4m2','r4m3', 'r4t1', 'r4t2',
'r4t3', 'v18q1', 'tamhog','tamviv','hhsize','hogar_nin',
'hogar_adul','hogar_mayor','hogar_total', 'bedrooms', 'qmobilephone']
hh_cont = ['v2a1', 'dependency', 'edjefe', 'edjefa', 'meaneduc', 'overcrowding']<define_variables>
regr = RandomForestRegressor(max_depth=10, min_samples_split=5, random_state=0,
n_estimators=500)
 scores = cross_val_score(regr, X, y, scoring="neg_mean_squared_error", cv=10)
 rmse_scores = np.sqrt(-scores)
 print(rmse_scores.mean() )
sqr_ = ['SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe',
'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned', 'agesq']<define_variables>
cols = [col for col in test.columns if col not in ['id']]
X_test= test[cols].values
x = ind_bool + ind_ordered + id_ + hh_bool + hh_ordered + hh_cont + sqr_
print('There are no repeats: ', np.all(np.array(list(Counter(x ).values())) == 1))
print('We covered every variable: ', len(x)== data.shape[1] )<define_variables>
regr.fit(X,y)
 y_pred = regr.predict(X_test )
heads = data.loc[data['parentesco1'] == 1, :]
heads = heads[id_ + hh_bool + hh_cont + hh_ordered]
heads.shape<filter>
y_pred=np.expm1(y_pred)
 pd.DataFrame({'id': test.id, 'revenue': y_pred} ).to_csv('submission_RF.csv', index=False )
corr_matrix.loc[corr_matrix['tamhog'].abs() > 0.9, corr_matrix['tamhog'].abs() > 0.9]<drop_column>
import xgboost as xgb
import lightgbm as lgb
heads = heads.drop(columns = ['tamhog', 'hogar_total', 'r4t3'] )<filter>
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2 )
corr_matrix.loc[corr_matrix['coopele'].abs() > 0.9, corr_matrix['coopele'].abs() > 0.9]<feature_engineering>
params = {'num_leaves': 30,
'min_data_in_leaf': 20,
'objective': 'regression',
'max_depth': 5,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.2,
"verbosity": -1}
lgb_model = lgb.LGBMRegressor(**params, n_estimators = 20000, nthread = 4, n_jobs = -1)
 lgb_model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='rmse',
verbose=1000, early_stopping_rounds=200 )
elec = []
for i, row in heads.iterrows() :
if row['noelec'] == 1:
elec.append(0)
 elif row['coopele'] == 1:
elec.append(1)
 elif row['public'] == 1:
elec.append(2)
 elif row['planpri'] == 1:
elec.append(3)
 else:
elec.append(np.nan)
 heads['elec'] = elec
heads['elec-missing'] = heads['elec'].isnull() 
<groupby>
lgb_model.fit(X, y,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='rmse',
verbose=1000, early_stopping_rounds=200)
 y_pred=lgb_model.predict(X_test )
heads = heads.drop(columns = 'area2')
 heads.groupby('area1')['Target'].value_counts(normalize = True )<drop_column>
y_pred=np.expm1(y_pred)
 pd.DataFrame({'id': test.id, 'revenue': y_pred} ).to_csv('submission_LGB.csv', index=False )
heads['roof'] = np.argmax(np.array(heads[['etecho1', 'etecho2', 'etecho3']]),
axis = 1)
 heads = heads.drop(columns = ['etecho1', 'etecho2', 'etecho3'])
 heads['floor'] = np.argmax(np.array(heads[['eviv1', 'eviv2', 'eviv3']]),
axis = 1)
<prepare_output>
xgb_params = {'eta': 0.01,
'objective': 'reg:linear',
'max_depth': 7,
'subsample': 0.8,
'colsample_bytree': 0.8,
'eval_metric': 'rmse',
'seed': 11,
'silent': True}
xgb_model = xgb.XGBRegressor(**xgb_params, n_estimators = 20000,
nthread = 4, n_jobs = -1 )
counts = pd.DataFrame(heads.groupby(['walls+roof+floor'])['Target'].value_counts(normalize = True)).rename(columns = {'Target': 'Normalized Count'} ).reset_index()
counts.head()<feature_engineering>
xgb_model.fit(X, y,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='rmse',
verbose=1000, early_stopping_rounds=200 )
heads['warning'] = 1 *(heads['sanitario1'] + 
(heads['elec'] == 0)+
heads['pisonotiene'] +
heads['abastaguano'] + 
(heads['cielorazo'] == 0))<feature_engineering>
y_pred = xgb_model.predict(X_test )
heads['phones-per-capita'] = heads['qmobilephone'] / heads['tamviv']
heads['tablets-per-capita'] = heads['v18q1'] / heads['tamviv']
heads['rooms-per-capita'] = heads['rooms'] / heads['tamviv']
heads['rent-per-capita'] = heads['v2a1'] / heads['tamviv']<import_modules>
y_pred=np.expm1(y_pred)
 pd.DataFrame({'id': test.id, 'revenue': y_pred} ).to_csv('submission_XGB.csv', index=False )
from scipy.stats import spearmanr<merge>
model_gboost = GradientBoostingRegressor()
model_gboost.fit(X_train, y_train)
 y_pred = model_gboost.predict(X_test )
<drop_column><EOS> 
y_pred=np.expm1(y_pred)
 pd.DataFrame({'id': test.id, 'revenue': y_pred} ).to_csv('submission_GradientBoosting.csv', index=False )