Kaggle competition: Costa Rican Household Poverty Level Prediction
xgb = XGBClassifier(gamma=0, learning_rate=0.1, max_depth=100, n_estimators=100)
 xgb.fit(filter_X_train,Y_train)
 xgb_y_pred_train = xgb.predict(filter_X_train)
 xgb_y_pred_test = xgb.predict(filter_X_test)
 scores_xg = cross_val_score(estimator = rf, y = Y_train, X = filter_X_train, cv=5)
 print('Cross Validation Score:', np.mean(scores_xg))<train_model>
pd.options.display.max_columns = 150
df_sm = df.drop(columns = cat_cols)
 smote = SMOTE(random_state = 1234)
 X_sm, Y_sm = smote.fit_resample(df_sm.drop(columns = 'loan_status'), df_sm['loan_status'])
 print("After OverSampling, counts of label '1': {}".format(sum(Y_sm == 1)))
 print("After OverSampling, counts of label '0': {}".format(sum(Y_sm == 0)) )<split>
train = pd.read_csv('.. /input/train.csv')
 train.info()
train.head()
Y = X_sm['loss']
X = X_sm.drop(columns = 'loss')
 X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.2, random_state = 1234, stratify = Y_sm)
 print('training set = {} records, test set= {} records'.format(X_train.shape[0],X_test.shape[0]))<features_selection>
test = pd.read_csv('.. /input/test.csv')
 test.info()
test.head()
X_train_num = X_train.drop(columns = bi_col_name)
 selector = SelectKBest(score_func = f_regression, k = 170)
 selector.fit(X_train_num,Y_train)
 select_cols_sm = selector.get_support(indices = True)
 select_num_cols_sm = X_train_num.iloc[:,select_cols_sm]
select_num_col_name = select_num_cols_sm.columns
select_num_cols_sm.head()<features_selection>
train_na= pd.DataFrame(( train.isnull().sum().values),index=train.columns, columns=['isNA'] ).sort_values(by=['isNA'],ascending=False)
 if train_na.loc[train_na.loc[:,'isNA']>0,:].shape[0]>1 :
print("Train Dataset")
 print(train_na.loc[train_na.loc[:,'isNA']> 0,])
 else:
print('no NA in train set')
 test_na= pd.DataFrame(( test.isnull().sum().values),index=test.columns, columns=['isNA'] ).sort_values(by=['isNA'],ascending=False)
 if train_na.loc[train_na.loc[:,'isNA']>0,:].shape[0]>1 :
print("")
 print("Test Dataset")
 print(test_na.loc[test_na.loc[:,'isNA']> 0,])
 else:
print('no NA in test set' )
selector = RFE(LogisticRegression() , n_features_to_select=150, step=1, verbose = 2)
 selector = selector.fit(select_num_cols_sm, Y_train)
 select_cols_sm = selector.get_support(indices = True)
 select_cols_df_sm = select_num_cols_sm.iloc[:,select_cols_sm]
best_X_col_name_sm = select_cols_df_sm.columns
select_cols_df_sm.head()<concatenate>
train.loc[:,"meaneduc"].mode()
train.loc[train.loc[:,"meaneduc"].isnull() ==True,"meaneduc"] = 6
train.loc[train.loc[:,"SQBmeaned"].isnull() ==True,"SQBmeaned"] = 36
test.loc[:,"meaneduc"].mode()
test.loc[test.loc[:,"meaneduc"].isnull() ==True,"meaneduc"] = 6
test.loc[test.loc[:,"SQBmeaned"].isnull() ==True,"SQBmeaned"] = 36
train.loc[train.loc[:,"rez_esc"].isnull() ==True,"rez_esc"] = 0
train.loc[train.loc[:,"v18q1"].isnull() ==True,"v18q1"] = 0
train.loc[train.loc[:,"v2a1"].isnull() ==True,"v2a1"] = 0
test.loc[test.loc[:,"rez_esc"].isnull() ==True,"rez_esc"] = 0
test.loc[test.loc[:,"v18q1"].isnull() ==True,"v18q1"] = 0
test.loc[test.loc[:,"v2a1"].isnull() ==True,"v2a1"] = 0
def filter_x_df_sm(x):
df = x.copy()
all_filter_col = []
for keep in select_cat_col_name[select_cat_col_name.isin(keep_cols)]:
filter_col = [col for col in df.columns if col.startswith(str(keep)) ]
for col in filter_col :
if col not in keep_cols:
all_filter_col.append(col)
 new_df = pd.concat([df[best_X_col_name_sm],df[all_filter_col]],axis = 1)
 return new_df<categorify>
train_na= pd.DataFrame(( train.isnull().sum().values),index=train.columns, columns=['isNA'] ).sort_values(by=['isNA'],ascending=False)
 if train_na.loc[train_na.loc[:,'isNA']>0,:].shape[0]>1 :
train_na.loc[train_na.loc[:,'isNA']> 0,]
else:
print('No NA in train set')
 test_na= pd.DataFrame(( test.isnull().sum().values),index=test.columns, columns=['isNA'] ).sort_values(by=['isNA'],ascending=False)
 if train_na.loc[train_na.loc[:,'isNA']>0,:].shape[0]>1 :
test_na.loc[test_na.loc[:,'isNA']> 0,]
else:
print('No NA in test set' )
filter_X_train_sm = filter_x_df_sm(X_train)
 filter_X_test_sm = filter_x_df_sm(X_test )<normalization>
target_Discrepancy=(train.groupby('idhogar')['Target'].nunique() >1)
 num_unique_households = train["idhogar"].unique().shape[0]
print('There are',target_Discrepancy.sum() ,'households with contradicting targets, out of', num_unique_households, 'households in the train dataset.' )
scaler = StandardScaler()
scaler.fit(filter_X_train_sm)
 X_train_scal_sm = scaler.fit_transform(filter_X_train_sm)
 X_test_scal_sm = scaler.fit_transform(filter_X_test_sm )<find_best_model_class>
for HH in HHID_Discrepancy:
Targets=(train.loc[train['idhogar']==HH,'Target'])
 if Targets.mode().shape[0] >1:
for i in Targets.index:
if train.loc[i,'parentesco1']==1:
HeadTarget= train.loc[i,"Target"]
for i in Targets.index:
train.loc[i,'Target']=HeadTarget
elif Targets.mode().shape[0]==1:
for i in Targets.index:
TrueTarget=int(Targets.mode())
 train.loc[i,'Target']=TrueTarget
logisticRegr_sm = LogisticRegression()
logisticRegr_sm = logisticRegr_sm.fit(X_train_scal_sm, Y_train)
 Lr_y_pred_train_sm = logisticRegr_sm.predict(X_train_scal_sm)
 Lr_y_pred_test_sm = logisticRegr_sm.predict(X_test_scal_sm)
 scores_lr_sm = cross_val_score(estimator = logisticRegr_sm, y = Y_train, X = X_train_scal_sm, cv=5)
 print('Cross Validation Score:', np.mean(scores_lr_sm))<count_values>
target_Discrepancy=(train.groupby('idhogar')['Target'].nunique() >1)
 print('There are ',target_Discrepancy.sum() ,'households with contradicting targets, out of 2988 households in the train dataset')
 train.head()
train.shape
cat_cols = df.select_dtypes(exclude=['number'] ).columns.values
for col in cat_cols:
if df[col].value_counts().count() > 20000 :
print('Column {} has {} categories'.format(col,df[col].value_counts().count()))<features_selection>
train=train.drop(columns=train.columns[133:142],axis=1)
 test=test.drop(columns=test.columns[133:142],axis=1 )
featurename = filter_X_train.columns
importances = list(rf.feature_importances_)
 feature_importances = [(feature, round(importance, 3)) for feature, importance in zip(featurename, importances)]
feature_importances = sorted(feature_importances, key = lambda x: x[1], reverse = True)
 print('Top 50 Importance Features
')
 [print('Variable: {} Importance Score: {}'.format(*pair)) for pair in feature_importances[:50]];<load_from_csv>
numColumnsHH= ["v2a1","rooms","v18q1","qmobilephone","r4h3","r4m3","r4t3","tamhog","hhsize","hogar_nin","hogar_adul",
"hogar_mayor","hogar_total","meaneduc","bedrooms","overcrowding"]
test_df = pd.read_csv('.. /input/loan-default-prediction/test_v2.csv.zip')
 test_df.head()<categorify>
wall = ["paredblolad","paredzocalo","paredpreb","pareddes","paredmad","paredzinc","paredfibras","paredother"]
floor = ["pisomoscer","pisocemento","pisoother","pisonatur","pisonotiene","pisomadera"]
roof = ['techozinc','techoentrepiso','techocane','techootro', 'cielorazo']
water = ["abastaguadentro","abastaguafuera","abastaguano"]
electric = ["public","planpri","noelec","coopele"]
toilet = ['sanitario1','sanitario2','sanitario3','sanitario5','sanitario6']
energy = ['energcocinar1','energcocinar2','energcocinar3','energcocinar4']
rubbish = ['elimbasu1','elimbasu2','elimbasu3','elimbasu4','elimbasu5','elimbasu6']
wall_cond = ['epared1','epared2','epared3']
roof_cond = ['etecho1','etecho2','etecho3']
floor_cond = ['eviv1','eviv2','eviv3']
col_types = [wall, floor, roof, water, electric, toilet, energy, rubbish, wall_cond, roof_cond, floor_cond]
titles = ['wall', 'floor', 'roof', 'water', 'electric', 'toilet', 'energy', 'rubbish', 'wall condition', 'roof condition', 'floor condition']
fig, ax =plt.subplots(len(col_types)//3 + 1, 3, figsize=(30,10))
plt.subplots_adjust(top=2.0)
 x=0
y=0
for i, cols in enumerate(col_types):
if(i>0)&(i % 3==0):
x=x+1
y=0
title = titles[i]
plot_df = pd.melt(cond_data, id_vars=['Target'], value_vars=cols ).groupby(["Target","variable"] ).apply(lambda x: np.mean(x))
plot_df["variable"] = plot_df.index.get_level_values(1)
 plot_df = plot_df.rename(columns = dict(value="Percentage Frequency", variable = "Attribute"))
sns.barplot(x="Attribute", y = "Percentage Frequency", hue="Target", data=plot_df, ax=ax[x,y])
 ax[x,y].title.set_text(title)
 y=y+1
encoder = ce.BinaryEncoder(cols = keep_cols)
 bi_enc_df = encoder.fit_transform(test_df[keep_cols])
 bi_col_name = bi_enc_df.columns
test_df = pd.concat([test_df,bi_enc_df],axis = 1)
 test_df.head()<concatenate>
geoTrain= train.loc[train['parentesco1'] == 1, :].copy()
geoTrain['lugar'] = np.argmax(np.array(cond_data[["lugar1","lugar2","lugar3","lugar4","lugar5","lugar6"]]),
axis = 1)
 geoTrain['lugar']=geoTrain['lugar'].replace({5:6,4:5,3:4,2:3,1:2,0:1})
select_test_df = pd.concat([test_df['id'],test_df[filter_X_train.columns]],axis = 1)
 select_test_df.head()<count_missing_values>
wall = ["paredblolad","paredzocalo","paredpreb","pareddes","paredmad","paredzinc","paredfibras","paredother"]
floor = ["pisomoscer","pisocemento","pisoother","pisonatur","pisonotiene","pisomadera"]
roof = ['techozinc','techoentrepiso','techocane','techootro', 'cielorazo']
water = ["abastaguadentro","abastaguafuera","abastaguano"]
electric = ["public","planpri","noelec","coopele"]
toilet = ['sanitario1','sanitario2','sanitario3','sanitario5','sanitario6']
energy = ['energcocinar1','energcocinar2','energcocinar3','energcocinar4']
rubbish = ['elimbasu1','elimbasu2','elimbasu3','elimbasu4','elimbasu5','elimbasu6']
wall_cond = ['epared1','epared2','epared3']
roof_cond = ['etecho1','etecho2','etecho3']
floor_cond = ['eviv1','eviv2','eviv3']
col_types = [wall, floor, roof, water, electric, toilet, energy, rubbish, wall_cond, roof_cond, floor_cond]
titles = ['wall', 'floor', 'roof', 'water', 'electric', 'toilet', 'energy', 'rubbish', 'wall condition', 'roof condition', 'floor condition']
fig, ax =plt.subplots(len(col_types)//3 + 1, 3, figsize=(30,10))
plt.subplots_adjust(top=2.0)
 x=0
y=0
for i, cols in enumerate(col_types):
if(i>0)&(i % 3==0):
x=x+1
y=0
title = titles[i]
plot_df = pd.melt(geoTrain, id_vars=['lugar'], value_vars=cols ).groupby(["lugar","variable"] ).apply(lambda x: np.mean(x))
plot_df["variable"] = plot_df.index.get_level_values(1)
 plot_df = plot_df.rename(columns = dict(value="Percentage Frequency", variable = "Attribute"))
sns.barplot(x="Attribute", y = "Percentage Frequency", hue="lugar", data=plot_df, ax=ax[x,y])
 ax[x,y].title.set_text(title)
 y=y+1
select_test_df.isnull().sum().value_counts()<correct_missing_values>
train['Adults']=train['hogar_adul']-train['hogar_mayor']
train['Dependents']=train['hogar_nin']+train['hogar_mayor']
train['Teenagers']=train['hogar_nin']-train['r4t1']
train['dependency']=train['Dependents']/train['Adults']
train['P_Adults']=train['Adults']/train['hogar_total']
train['P_Adults_Male']=train['r4h3']/train['hogar_total']
train['P_Adults_Female']=train['r4m3']/train['hogar_total']
train['P_Children']=train['hogar_nin']/train['hogar_total']
train['P_Seniors']=train['hogar_mayor']/train['hogar_total']
train['P_Teenagers']=train['Teenagers']/train['hogar_total']
train['RentHH']=train['v2a1']/train['hogar_total']
train['RentAdults'] = train['v2a1']/train['Adults']
train['RentAdults'] = train['RentAdults'].fillna(train['v2a1'])
 train['Tablet_PP']=train['v18q1']/train['hogar_total']
train['Phone_PP']=train['qmobilephone']/train['hogar_total']
train['Bedroom_PP']=train['bedrooms']/train['hogar_total']
train['Appliances']=train['refrig']+train['computer']+train['television']
train['Appliances_PP']=train['Appliances']/test['hogar_total']
train['HHS_Diff']=train['tamviv']-train['hhsize']
train["AdultsPerRoom"] = train['Adults']/train['rooms']
train["DependentsPerRoom"] = train['Dependents']/train['rooms']
train["TeenagersPerRoom"] = train['Teenagers']/train['rooms']
train["MalesPerRoom"] = train['r4h3']/train['rooms']
train["FemalesPerRoom"] = train['r4m3']/train['rooms']
train["BedroomPerRoom"] = train['bedrooms']/train['rooms']
train["RentPerRoom"] = train['v2a1']/train['rooms']
train["Schooling_Age"] = train["escolari"]/train["age"]
train['SchoolingProp'] = train['rez_esc']/train['escolari']
train["EduLevel"] = 0
train.loc[train["instlevel9"] == 1,"EduLevel"] = 6
train.loc[train["instlevel8"] == 1,"EduLevel"] = 5
train.loc[train["instlevel7"] == 1,"EduLevel"] = 3
train.loc[train["instlevel5"] == 1,"EduLevel"] = 2
train.loc[(train[["instlevel4","instlevel3","instlevel6"]].sum(axis = 1)> 0),"EduLevel"] = 1
train.head()
test['Adults']=test['hogar_adul']-test['hogar_mayor']
test['Dependents']=test['hogar_nin']+test['hogar_mayor']
test['Teenagers']=test['hogar_nin']-test['r4t1']
test['dependency']=test['Dependents']/test['Adults']
test['P_Adults']=test['Adults']/test['hogar_total']
test['P_Adults_Male']=test['r4h3']/test['hogar_total']
test['P_Adults_Female']=test['r4m3']/test['hogar_total']
test['P_Children']=test['hogar_nin']/test['hogar_total']
test['P_Seniors']=test['hogar_mayor']/test['hogar_total']
test['P_Teenagers']=test['Teenagers']/test['hogar_total']
test['RentHH']=test['v2a1']/test['hogar_total']
test['RentAdults']=test['v2a1']/test['Adults']
test['RentAdults'] = test['RentAdults'].fillna(test['v2a1'])
 test['Tablet_PP']=test['v18q1']/test['hogar_total']
test['Phone_PP']=test['qmobilephone']/test['hogar_total']
test['Bedroom_PP']=test['bedrooms']/test['hogar_total']
test['Appliances']=test['refrig']+test['computer']+test['television']
test['Appliances_PP']=test['Appliances']/test['hogar_total']
test['HHS_Diff']=test['tamviv']-test['hhsize']
test["AdultsPerRoom"] = test['Adults']/test['rooms']
test["DependentsPerRoom"] = test['Dependents']/test['rooms']
test["TeenagersPerRoom"] = test['Teenagers']/test['rooms']
test["MalesPerRoom"] = test['r4h3']/test['rooms']
test["FemalesPerRoom"] = test['r4m3']/test['rooms']
test["BedroomPerRoom"] = test['bedrooms']/test['rooms']
test["RentPerRoom"] = test['v2a1']/test['rooms']
test["Schooling_Age"] = test["escolari"]/test["age"]
test['SchoolingProp'] = test['rez_esc']/test['escolari']
test["EduLevel"] = 0
test.loc[test["instlevel9"] == 1,"EduLevel"] = 6
test.loc[test["instlevel8"] == 1,"EduLevel"] = 5
test.loc[test["instlevel7"] == 1,"EduLevel"] = 3
test.loc[test["instlevel5"] == 1,"EduLevel"] = 2
test.loc[(test[["instlevel4","instlevel3","instlevel6"]].sum(axis = 1)> 0),"EduLevel"] = 1
test.head()
numeric_cols = select_test_df.select_dtypes(include=['number'] ).columns.values
for col in numeric_cols:
if col in list(col_pct_miss_df.column_name):
med = df[col].median()
select_test_df[col] = select_test_df[col].fillna(med)
 not_numeric_cols = select_test_df.select_dtypes(exclude=['number'] ).columns.values
for col in not_numeric_cols:
mode = df[col].mode()
select_test_df[col] = select_test_df[col].fillna(mode[0] )<count_missing_values>
List_Mean = ['rez_esc', 'male', 'female', 'estadocivil1', 'estadocivil2', 'estadocivil3', 'estadocivil4', 'estadocivil5',
'estadocivil6', 'estadocivil7', 'parentesco2','parentesco3', 'parentesco4', 'parentesco5', 'parentesco6', 'parentesco7',
'parentesco8', 'parentesco9', 'parentesco10', 'parentesco11', 'parentesco12','instlevel1', 'instlevel2', 'instlevel3',
'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9','overcrowding']
List_Summary = ['age', 'escolari','dis','EduLevel', 'Schooling_Age']
trainGP = pd.DataFrame()
testGP = pd.DataFrame()
for item in List_Mean:
group_train_mean = train[item].groupby(train['idhogar'] ).mean()
group_test_mean = test[item].groupby(test['idhogar'] ).mean()
new_col = item + '_mean'
trainGP[new_col] = group_train_mean
testGP[new_col] = group_test_mean
for item in List_Summary:
for function in ['mean','std','min','max','sum']:
group_train = train[item].groupby(train['idhogar'] ).agg(function)
 group_test = test[item].groupby(test['idhogar'] ).agg(function)
 new_col = item + '_' + function
trainGP[new_col] = group_train
testGP[new_col] = group_test
trainGP['age_range']=trainGP['age_max']-trainGP['age_min']
testGP['age_range']=testGP['age_max']-testGP['age_min']
trainGP['escolari_range']=trainGP['escolari_max']-trainGP['escolari_min']
testGP['escolari_range']=testGP['escolari_max']-testGP['escolari_min']
trainGP["age_std"] = trainGP["age_std"].fillna(0)
 trainGP["escolari_std"] = trainGP["escolari_std"].fillna(0)
 trainGP["dis_std"] = trainGP["dis_std"].fillna(0)
 trainGP["EduLevel_std"] = trainGP["EduLevel_std"].fillna(0)
 trainGP["Schooling_Age_std"] = trainGP["Schooling_Age_std"].fillna(0)
 testGP["age_std"] = testGP["age_std"].fillna(0)
 testGP["escolari_std"] = testGP["escolari_std"].fillna(0)
 testGP["dis_std"] = testGP["dis_std"].fillna(0)
 testGP["EduLevel_std"] = testGP["EduLevel_std"].fillna(0)
 testGP["Schooling_Age_std"] = testGP["Schooling_Age_std"].fillna(0)
 trainGP.head()
testGP.head()
select_test_df.isnull().sum().value_counts()<correct_missing_values>
train["GoodCondition"] = train["paredblolad"] + \
train["pisomoscer"] + \
train["cielorazo"] + \
train["abastaguadentro"] + \
train["public"] + \
train["sanitario2"] + \
train["energcocinar2"] + \
train["elimbasu1"] + \
train["epared3"] + \
train["etecho3"] + \
train["eviv3"]
train["GoodCondition"] = train["GoodCondition"]/11
train["AverageCondition"] =(train["paredzocalo"] + train["paredpreb"])/2 + \
train["techozinc"] + \
train["abastaguafuera"] + \
train["coopele"] + \
train["sanitario3"] + \
train["energcocinar3"] + \
train["elimbasu2"] + \
train["epared2"] + \
train["etecho2"] + \
train["eviv2"]
train["AverageCondition"] = train["AverageCondition"]/10
train["BadCondition"] =(train["pareddes"] + train["paredmad"] + train["paredzinc"])/3 + \ 
(train["pisocemento"] + train["pisonotiene"] + train["pisomadera"])/3 + \
train["abastaguano"] + \
train["noelec"] + \ 
(train["sanitario1"] + train["sanitario5"])/2 + \ 
(train["energcocinar1"] + train["energcocinar4"])/2 + \
train["elimbasu3"] + \
train["epared1"] + \
train["etecho1"] + \
train["eviv1"]
train["BadCondition"] = train["BadCondition"]/10
test["GoodCondition"] = test["paredblolad"] + \
test["pisomoscer"] + \
test["cielorazo"] + \
test["abastaguadentro"] + \
test["public"] + \
test["sanitario2"] + \
test["energcocinar2"] + \
test["elimbasu1"] + \
test["epared3"] + \
test["etecho3"] + \
test["eviv3"]
test["GoodCondition"] = test["GoodCondition"]/11
test["AverageCondition"] =(test["paredzocalo"] + test["paredpreb"])/2 + \
test["techozinc"] + \
test["abastaguafuera"] + \
test["coopele"] + \
test["sanitario3"] + \
test["energcocinar3"] + \
test["elimbasu2"] + \
test["epared2"] + \
test["etecho2"] + \
test["eviv2"]
test["AverageCondition"] = test["AverageCondition"]/10
test["BadCondition"] =(test["pareddes"] + test["paredmad"] + test["paredzinc"])/3 + \ 
(test["pisocemento"] + test["pisonotiene"] + test["pisomadera"])/3 + \
test["abastaguano"] + \
test["noelec"] + \ 
(test["sanitario1"] +test["sanitario5"])/2 + \ 
(test["energcocinar1"] + test["energcocinar4"])/2 + \
test["elimbasu3"] + \
test["epared1"] + \
test["etecho1"] + \
test["eviv1"]
test["BadCondition"] = test["BadCondition"]/10
select_test_df[columns_with_nan] = select_test_df[columns_with_nan].fillna(0 )<predict_on_test>
trainGP = trainGP.reset_index()
testGP = testGP.reset_index()
trainML = pd.merge(train, trainGP, on='idhogar')
 testML = pd.merge(test, testGP, on='idhogar')
 trainML = trainML.query("parentesco1==1")
 trainML.fillna(value=0, inplace=True)
 testML.fillna(value=0, inplace=True)
 trainML = trainML.replace([np.inf, -np.inf], 0)
 testML = testML.replace([np.inf, -np.inf], 0)
 train.shape, test.shape, trainGP.shape, testGP.shape, trainML.shape, testML.shape
test_df_rf = test_df.copy()
test_df_rf['loss'] = rf.predict(select_test_df.loc[:,select_test_df.columns != 'id'])
 test_df_rf.head()<save_to_csv>
trainML.isna().any().any() ,testML.isna().any().any()
sample_submission = test_df_rf[['id','loss']]
sample_submission.to_csv('sample_submission_rf.csv', index = False )<set_options>
submission = testML.copy() [['Id']]
%matplotlib inline
le = preprocessing.LabelEncoder()
style.use('ggplot')
 warnings.filterwarnings('ignore')
 gc.enable()
!ls.. /input/
!ls.. /input/robots-best-submission
print("Ready !" )<load_from_csv>
trainML.drop(columns=['idhogar','Id','tamhog','r4t3','hhsize','hogar_adul','edjefe','edjefa'],inplace=True)
 testML.drop(columns=['idhogar','Id','tamhog','r4t3','hhsize','hogar_adul','edjefe','edjefa'],inplace=True)
data = pd.read_csv('.. /input/career-con-2019/X_train.csv')
 tr = pd.read_csv('.. /input/career-con-2019/X_train.csv')
 sub = pd.read_csv('.. /input/career-con-2019/sample_submission.csv')
 test = pd.read_csv('.. /input/career-con-2019/X_test.csv')
 target = pd.read_csv('.. /input/career-con-2019/y_train.csv')
 print("Data is ready !!" )<count_values>
correlation=trainML.corr()
correlation = correlation['Target'].sort_values(ascending=False)
 print(f'The most 20 positively correlated feature:
{correlation.head(20)}')
 print('*'*50)
 print(f'The most 20 negatively correlated feature:
{correlation.tail(20)}' )
len(data.measurement_number.value_counts() )<sort_values>
columns_remove = ['elimbasu5','estadocivil1','parentesco1','parentesco2','parentesco3','parentesco4','parentesco5','parentesco6','parentesco7','parentesco8','parentesco9','parentesco10','parentesco11','parentesco12']
trainML.drop(columns=columns_remove, inplace=True)
 testML.drop(columns=columns_remove, inplace=True )
totalt = data.isnull().sum().sort_values(ascending=False)
 percent =(data.isnull().sum() /data.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([totalt, percent], axis=1, keys=['Total', 'Percent'])
 print("Missing Data at Training")
 missing_data.tail()<sort_values>
X = trainML.drop(columns=['Target'])
 y = trainML['Target']
Xtest = testML.copy()
totalt = test.isnull().sum().sort_values(ascending=False)
 percent =(test.isnull().sum() /data.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([totalt, percent], axis=1, keys=['Total', 'Percent'])
 print("Missing Data at Test")
 missing_data.tail()<count_unique_values>
svm_model = SVC(gamma='auto')
 svm_model.fit(X, y )
target.groupby('group_id' ).surface.nunique().max()<count_unique_values>
test_prediction_svm = svm_model.predict(Xtest )
target['group_id'].nunique()<set_options>
rf_model = RandomForestClassifier(n_estimators=100)
 rf_model.fit(X,y)
del serie1
gc.collect()<feature_engineering>
test_prediction_rf = rf_model.predict(Xtest )
series_dict = {}
for series in(data['series_id'].unique()):
series_dict[series] = data[data['series_id'] == series]<set_options>
def evaluate_macroF1_lgb(truth, predictions):
predictions = np.resize(predictions, new_shape=(4,len(truth)))
 pred_labels = predictions.argmax(axis=0)
 f1 = f1_score(truth, pred_labels, average='macro')
 eval_name, val, is_higher_better = "macroF1", f1, True
return(eval_name, val, is_higher_better)
 clf = lgb.LGBMClassifier(max_depth=12, learning_rate=0.005, objective='multiclass',
random_state=69, silent=True, metric='None',
n_jobs=4, n_estimators=2500, class_weight='balanced',
colsample_bytree = 0.89, min_child_samples = 80, num_leaves = 14, subsample = 0.96)
 clf
del series_dict
gc.collect()<load_from_csv>
kfold = 5
kf = StratifiedKFold(n_splits=kfold, shuffle=True, random_state = 123 )
train_x = pd.read_csv('.. /input/career-con-2019/X_train.csv')
 train_y = pd.read_csv('.. /input/career-con-2019/y_train.csv' )<create_dataframe>
val_predictions_logloss = []
test_predictions_logloss = []
test_predictions_logloss_proba = []
for train_index, val_index in kf.split(X, y):
print("=======")
 X_train, X_val = X.iloc[train_index], X.iloc[val_index]
y_train, y_val = y.iloc[train_index], y.iloc[val_index]
clf.fit(X_train, y_train, eval_set=[(X_val, y_val)], eval_metric="logloss",
early_stopping_rounds=400, verbose=200)
 val_predictions_logloss.append(clf.predict(X_val))
test_predictions_logloss.append(clf.predict(Xtest))
test_predictions_logloss_proba.append(clf.predict_proba(Xtest))
def prepare_data(t):
def f(d):
d=d.sort_values(by=['measurement_number'])
 return pd.DataFrame({
'lx':[ d['linear_acceleration_X'].values ],
'ly':[ d['linear_acceleration_Y'].values ],
'lz':[ d['linear_acceleration_Z'].values ],
'ax':[ d['angular_velocity_X'].values ],
'ay':[ d['angular_velocity_Y'].values ],
'az':[ d['angular_velocity_Z'].values ],
})
 t= t.groupby('series_id' ).apply(f)
 def mfft(x):
return [ x/math.sqrt(128.0)for x in np.absolute(np.fft.fft(x)) ][1:65]
t['lx_f']=[ mfft(x)for x in t['lx'].values ]
t['ly_f']=[ mfft(x)for x in t['ly'].values ]
t['lz_f']=[ mfft(x)for x in t['lz'].values ]
t['ax_f']=[ mfft(x)for x in t['ax'].values ]
t['ay_f']=[ mfft(x)for x in t['ay'].values ]
t['az_f']=[ mfft(x)for x in t['az'].values ]
return t<merge>
X2 = X.iloc[:,top_100_indices]
Xtest2 = testML.copy().iloc[:,top_100_indices]
test_predictions2 = []
test_predictions_proba2 = []
for train_index, val_index in kf.split(X2, y):
print("=======")
 X_train, X_val = X2.iloc[train_index], X2.iloc[val_index]
y_train, y_val = y.iloc[train_index], y.iloc[val_index]
clf.fit(X_train, y_train, eval_set=[(X_val, y_val)], eval_metric="logloss",
early_stopping_rounds=400, verbose=200)
 test_predictions2.append(clf.predict(Xtest2))
test_predictions_proba2.append(clf.predict_proba(Xtest2))
t=prepare_data(train_x)
 t=pd.merge(t,train_y[['series_id','surface','group_id']],on='series_id')
 t=t.rename(columns={"surface": "y"} )<create_dataframe>
submission_svm = submission.copy()
submission_svm['Target'] = np.array(test_prediction_svm ).astype(int)
 submission_svm.to_csv("submission_01.csv", index=False)
 print(submission_svm['Target'].value_counts() /submission_svm['Target'].value_counts().sum())
 print("The Macro F1 Score(Test)is: 0.28218" )
def aggf(d, feature):
va= np.array(d[feature].tolist())
 mean= sum(va)/va.shape[0]
var= sum([(va[i,:]-mean)**2 for i in range(va.shape[0])])/va.shape[0]
dev= [ math.sqrt(x)for x in var ]
return pd.DataFrame({
'mean': [ mean ],
'dev' : [ dev ],
})
 display={
'hard_tiles_large_space':'r-.',
'concrete':'g-.',
'tiled':'b-.',
'fine_concrete':'r-',
'wood':'g-',
'carpet':'b-',
'soft_pvc':'y-',
'hard_tiles':'r--',
'soft_tiles':'g--',
}<set_options>
submission_rf = submission.copy()
submission_rf['Target'] = np.array(test_prediction_rf ).astype(int)
 submission_rf.to_csv("submission_02.csv", index=False)
 print(submission_rf['Target'].value_counts() /submission_rf['Target'].value_counts().sum())
 print("The Macro F1 Score(Test)is: 0.37434" )
del train_x, train_y
gc.collect()<normalization>
submission_full_logloss = submission.copy()
voted_test_predictions = np.apply_along_axis(lambda x: np.bincount(x ).argmax() , axis=0, arr=test_predictions_logloss)
 submission_full_logloss['Target'] = np.array(voted_test_predictions ).astype(int)
 submission_full_logloss.to_csv("submission_03.csv", index=False)
 print(submission_full_logloss['Target'].value_counts() /submission_full_logloss['Target'].value_counts().sum())
 print("The Macro F1 Score(Test)is: 0.43970" )
<feature_engineering><EOS> 
submission_top100_logloss = submission.copy()
voted_test_predictions = np.apply_along_axis(lambda x: np.bincount(x ).argmax() , axis=0, arr=test_predictions2)
 submission_top100_logloss['Target'] = np.array(voted_test_predictions ).astype(int)
 submission_top100_logloss.to_csv("submission_04.csv", index=False)
 print(submission_top100_logloss['Target'].value_counts() /submission_top100_logloss['Target'].value_counts().sum())
 print("The Macro F1 Score(Test)is: 0.44019" )