Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<define_variables>
sns.set()
%matplotlib inline
labels = {'v2a1':' Monthly rent payment','hacdor':' =1 Overcrowding by bedrooms','rooms':' number of all rooms in the house','hacapo':' =1 Overcrowding by rooms','v14a':' =1 has toilet in the household','refrig':' =1 if the household has refrigerator','v18q':' owns a tablet','v18q1':' number of tablets household owns','r4h1':' Males younger than 12 years of age','r4h2':' Males 12 years of age and older','r4h3':' Total males in the household','r4m1':' Females younger than 12 years of age','r4m2':' Females 12 years of age and older','r4m3':' Total females in the household','r4t1':' persons younger than 12 years of age','r4t2':' persons 12 years of age and older','r4t3':' Total persons in the household','tamhog':' size of the household','tamviv':' TamViv','escolari':' years of schooling','rez_esc':' Years behind in school','hhsize':' household size','paredblolad':' =1 if predominant material on the outside wall is block or brick','paredzocalo':' "=1 if predominant material on the outside wall is socket(wood zinc or absbesto"','paredpreb':' =1 if predominant material on the outside wall is prefabricated or cement','pareddes':' =1 if predominant material on the outside wall is waste material','paredmad':' =1 if predominant material on the outside wall is wood','paredzinc':' =1 if predominant material on the outside wall is zink','paredfibras':' =1 if predominant material on the outside wall is natural fibers','paredother':' =1 if predominant material on the outside wall is other','pisomoscer':' "=1 if predominant material on the floor is mosaic ceramic terrazo"','pisocemento':' =1 if predominant material on the floor is cement','pisoother':' =1 if predominant material on the floor is other','pisonatur':' =1 if predominant material on the floor is natural material','pisonotiene':' =1 if no floor at the household','pisomadera':' =1 if predominant material on the floor is wood','techozinc':' =1 if predominant material on the roof is metal foil or zink','techoentrepiso':' "=1 if predominant material on the roof is fiber cement mezzanine "','techocane':' =1 if predominant material on the roof is natural fibers','techootro':' =1 if predominant material on the roof is other','cielorazo':' =1 if the house has ceiling','abastaguadentro':' =1 if water provision inside the dwelling','abastaguafuera':' =1 if water provision outside the dwelling','abastaguano':' =1 if no water provision','public':' "=1 electricity from CNFL ICE ESPH/JASEC"','planpri':' =1 electricity from private plant','noelec':' =1 no electricity in the dwelling','coopele':' =1 electricity from cooperative','sanitario1':' =1 no toilet in the dwelling','sanitario2':' =1 toilet connected to sewer or cesspool','sanitario3':' =1 toilet connected to septic tank','sanitario5':' =1 toilet connected to black hole or letrine','sanitario6':' =1 toilet connected to other system','energcocinar1':' =1 no main source of energy used for cooking(no kitchen)','energcocinar2':' =1 main source of energy used for cooking electricity','energcocinar3':' =1 main source of energy used for cooking gas','energcocinar4':' =1 main source of energy used for cooking wood charcoal','elimbasu1':' =1 if rubbish disposal mainly by tanker truck','elimbasu2':' =1 if rubbish disposal mainly by botan hollow or buried','elimbasu3':' =1 if rubbish disposal mainly by burning','elimbasu4':' =1 if rubbish disposal mainly by throwing in an unoccupied space','elimbasu5':' "=1 if rubbish disposal mainly by throwing in river creek or sea"','elimbasu6':' =1 if rubbish disposal mainly other','epared1':' =1 if walls are bad','epared2':' =1 if walls are regular','epared3':' =1 if walls are good','etecho1':' =1 if roof are bad','etecho2':' =1 if roof are regular','etecho3':' =1 if roof are good','eviv1':' =1 if floor are bad','eviv2':' =1 if floor are regular','eviv3':' =1 if floor are good','dis':' =1 if disable person','male':' =1 if male','female':' =1 if female','estadocivil1':' =1 if less than 10 years old','estadocivil2':' =1 if free or coupled uunion','estadocivil3':' =1 if married','estadocivil4':' =1 if divorced','estadocivil5':' =1 if separated','estadocivil6':' =1 if widow/er','estadocivil7':' =1 if single','parentesco1':' =1 if household head','parentesco2':' =1 if spouse/partner','parentesco3':' =1 if son/doughter','parentesco4':' =1 if stepson/doughter','parentesco5':' =1 if son/doughter in law','parentesco6':' =1 if grandson/doughter','parentesco7':' =1 if mother/father','parentesco8':' =1 if father/mother in law','parentesco9':' =1 if brother/sister','parentesco10':' =1 if brother/sister in law','parentesco11':' =1 if other family member','parentesco12':' =1 if other non family member','idhogar':' Household level identifier','hogar_nin':' Number of children 0 to 19 in household','hogar_adul':' Number of adults in household','hogar_mayor':'
fp1 = os.path.join(".. ", "input", "train.csv")
 fp2 = os.path.join(".. ", "input", "test.csv" )<load_from_csv>
df = pd.read_csv(".. /input/train.csv")
 df_test = pd.read_csv(".. /input/test.csv")
 df['Target0']=df['Target']-1
locations = ['lugar1','lugar2','lugar3','lugar4','lugar5','lugar6']
le = LabelEncoder()
df['idhogarnum'] = df[['idhogar']].apply(le.fit_transform)
 df_test['idhogarnum'] = df_test[['idhogar']].apply(le.fit_transform )
train = pd.read_csv(fp1, index_col=0)
 train.head()<load_from_csv>
df = pd.concat([df,pd.get_dummies(df['Target'], prefix='Target')],axis=1)
 df.head()
dataframes = [df,df_test]
test = pd.read_csv(fp2, index_col=0)
 test.head()<filter>
df.groupby('Target')['Target'].count()
T1p = 755.0/9557.0
T2p = 1597.0/9557.0
T3p = 1209.0/9557.0
T4p = 5996.0/9557.0
RT1p = 1.0/T1p
RT2p = 1.0/T2p
RT3p = 1.0/T3p
RT4p = 1.0/T4p
print(RT1p,RT2p,RT3p,RT4p)
 df['targetweight'] = 0.0
df.loc[df['Target']==1,'targetweight']=RT1p
df.loc[df['Target']==2,'targetweight']=RT2p
df.loc[df['Target']==3,'targetweight']=RT3p
df.loc[df['Target']==4,'targetweight']=RT4p
train = train[(train.trip_duration < 5000)]<filter>
individual_variables = ['age','mobilephone','escolari','mobilephone','male','female','dis',
'instlevel1','instlevel2','instlevel3','instlevel4','instlevel5',
'instlevel6','instlevel7','instlevel8','instlevel9','estadocivil1',
'estadocivil2','estadocivil3','estadocivil4','estadocivil5','estadocivil6',
'estadocivil7','parentesco1','parentesco2','parentesco3','parentesco4',
'parentesco5','parentesco6','parentesco7','parentesco8','parentesco9',
'parentesco10','parentesco11','parentesco12']
for tdf in dataframes:
tdf['indeduindex25'] = 0
tdf.loc[tdf['age']>=25,'indeduindex25'] =(tdf['escolari'] / 15.0 +(7/18.0)) /2.0
tdf.loc[tdf['indeduindex25'].isna() ,'indeduindex25'] = 0
individual_variables.append('indeduindex25')
 tdf['agecentschool'] = tdf['age']-7
tdf['attendSchool'] = 0
tdf['attendSchool'] = np.where(tdf['escolari']>0,1,0)
 tdf['attendSchool'] = np.where(( tdf['age'] >=6)&(tdf['age'] <=13), tdf['attendSchool'],0)
 individual_variables.append('attendSchool')
 tdf['behindSchool'] = 0
tdf['behindSchool'] = np.where(tdf['agecentschool'] >tdf['escolari'],1,0)
 tdf['behindSchool'] = np.where(( tdf['age'] >=6)&(tdf['age'] <=13), tdf['behindSchool'],0)
 individual_variables.append('behindSchool')
 tdf['behindSchool18'] = 0
tdf['behindSchool18'] = np.where(tdf['agecentschool'] >tdf['escolari'],1,0)
 tdf['behindSchool18'] = np.where(( tdf['age'] >=6)&(tdf['age'] <=18), tdf['behindSchool18'],0)
 individual_variables.append('behindSchool18')
 tdf['portionofhh'] = 0
tdf['portionofhh'] = 1.0 / tdf['hhsize']
tdf.loc[tdf['parentesco1']==1,'portionofhh']=1.0
individual_variables.append('portionofhh')
 pd.set_option('display.max_columns', None)
 individual_variables = list(set(individual_variables))
train = train.loc[(train['pickup_longitude'] > -75)&(train['pickup_longitude'] < -73)]
train = train.loc[(train['pickup_latitude'] > 40)&(train['pickup_latitude'] < 41)]<filter>
household_variables = ['lugar1','lugar2','lugar3','lugar4','lugar5','lugar6','area1','area2',
'paredblolad','paredzocalo','paredpreb','pareddes','paredmad','paredzinc',
'paredfibras','paredother','pisomoscer','pisocemento','pisoother','pisonatur',
'pisonotiene','pisomadera','techozinc','techoentrepiso','techocane','techootro',
'cielorazo','abastaguadentro','abastaguafuera','abastaguano','public','planpri',
'noelec','coopele','sanitario1','sanitario2','sanitario3','sanitario5','sanitario6',
'energcocinar1','energcocinar2','energcocinar3','energcocinar4','elimbasu1','elimbasu2',
'elimbasu3','elimbasu4','elimbasu5','elimbasu6','epared1','epared2','epared3','etecho1',
'etecho2','etecho3','eviv1','eviv2','eviv3','tipovivi1','tipovivi2','tipovivi3','tipovivi4','tipovivi5',
'v18q1','qmobilephone','tamviv','tamhog','r4h1','r4h2','r4h3','r4m1','r4m2','r4m3','bedrooms', 'computer', 'hacapo', 'hacdor', 'hogar_adul', 'hogar_mayor', 'hogar_nin', 'r4t1',
'r4t2','r4t3', 'refrig', 'rooms', 'television', 'v14a', 'v18q']
for tdf in dataframes:
tdf.loc[tdf['v2a1'].isna() ,'v2a1'] = 0
tdf.loc[tdf['v18q1'].isna() ,'v18q1'] = 0
tdf.loc[tdf['qmobilephone'].isna() ,'qmobilephone'] = 0
conditions = [(tdf['v2a1']==0),(tdf['v2a1']!=0)]
choices = [0,np.log(tdf['v2a1']+.0001)]
tdf['lv2a1']=np.select(conditions, choices)
 household_variables.append('lv2a1')
 tdf['medhhage'] = tdf['age'].groupby(tdf['idhogar'] ).transform('median')
 household_variables.append('medhhage')
 tdf['eduindex'] = tdf[tdf['age']>=25]['escolari'].groupby(tdf['idhogar'] ).transform('mean')
 tdf['eduindex'] =(tdf['eduindex'] / 15.0 +(8.3/18.0)) /2.0
tdf['eduindex'] = tdf['eduindex'].groupby(tdf['idhogar'] ).transform('mean')
 tdf.loc[tdf['eduindex'].isna() ,'eduindex'] = 0
household_variables.append('eduindex')
 tdf['atleast8'] = 0
tdf['atleast8'] = tdf[tdf['escolari']>=8]['escolari'].groupby(tdf['idhogar'] ).transform('count')
 tdf['atleast8'] = tdf['atleast8'].groupby(tdf['idhogar'] ).transform('max')
 tdf['atleast8'] = np.where(tdf['atleast8']>=1, 1, 0)
 household_variables.append('atleast8')
 tdf['atleast8F'] = 0
tdf['atleast8F'] = tdf[(tdf['escolari']>=6)&(tdf['female']>=1)]['escolari'].groupby(tdf['idhogar'] ).transform('count')
 tdf['atleast8F'] = tdf['atleast8F'].groupby(tdf['idhogar'] ).transform('max')
 tdf['atleast8F'] = np.where(tdf['atleast8F']>=1, 1, 0)
 household_variables.append('atleast8F')
 tdf['nhhlt14'] = 0
tdf['nhhlt14'] = tdf[tdf['age']<=14]['age'].groupby(tdf['idhogar'] ).transform('count')
 tdf['nhhlt14'] = tdf['nhhlt14'].groupby(tdf['idhogar'] ).transform('max')
 tdf['nhhlt14'] = np.where(tdf['nhhlt14']>=1, tdf['nhhlt14'], 0)
 household_variables.append('nhhlt14')
 tdf['perhhlt14'] = 0.0
tdf['perhhlt14'] = tdf['nhhlt14'] / tdf['hhsize']
household_variables.append('perhhlt14')
 tdf['perschoolage'] = 0.0
tdf['perschoolage'] = tdf[(tdf['age'] >=6)&(tdf['age'] <=13)].groupby(tdf['idhogar'] ).transform('count')
 tdf['perschoolage'] = tdf['perschoolage'].groupby(tdf['idhogar'] ).transform('max')
 tdf['perschoolage'] = np.where(tdf['perschoolage']>=1, tdf['perschoolage'], 0)
 tdf['perschoolage'] = tdf['perschoolage'] / tdf['hhsize']
household_variables.append('perschoolage')
 tdf['cupara'] = tdf['hogar_adul']+(0.5*tdf['nhhlt14'])
 household_variables.append('cupara')
 tdf['assets2'] = 0
tdf.loc[(tdf['v18q']<=1)&(tdf['qmobilephone']<=1)&(tdf['computer']==0)&(tdf['television']==0),'assets2'] = 1
household_variables.append('assets2')
 tdf['assets3'] = 0
tdf.loc[(tdf['v18q']<=1)&(tdf['qmobilephone']>=1)&(tdf['computer']==0)&(tdf['television']==1),'assets3'] = 1
household_variables.append('assets3')
 tdf['assets3b'] = 0
tdf.loc[(tdf['v18q']<=1)&(tdf['qmobilephone']>=1)&(tdf['computer']==0)&(tdf['television']==0),'assets3b'] = 1
household_variables.append('assets3b')
 tdf['assets4'] = 0
tdf.loc[(tdf['v18q']>=1)&(tdf['qmobilephone']>=1)&(tdf['computer']==1)&(tdf['television']==1),'assets4'] = 1
household_variables.append('assets4')
 conditions = [(tdf['parentesco1']==1)&(tdf['estadocivil4']==1),(tdf['parentesco1']==1)&(tdf['estadocivil5']==1), 
(tdf['parentesco1']==1)&(tdf['estadocivil6']==1), 
(tdf['parentesco1']==1)&(tdf['estadocivil7']==1),(tdf['parentesco1']==0)]
choices = [1,1,1,1,0]
tdf['headmarital']=np.select(conditions, choices)
 tdf['headmarital'] = tdf['headmarital'].groupby(tdf['idhogar'] ).transform('count')
 tdf['headmarital'] = np.where(tdf['headmarital']>=1, 1, 0)
 household_variables.append('headmarital')
 conditions = [(tdf['hogar_adul']==0),(tdf['hogar_adul']>0)]
choices = [0, tdf['hogar_nin']/tdf['hogar_adul']]
tdf['radultchild']=np.select(conditions, choices)
 household_variables.append('radultchild')
 conditions = [(tdf['tamhog']==0),(tdf['tamhog']>0)]
choices = [0, tdf['bedrooms']/tdf['tamhog']]
tdf['bedperperson']=np.select(conditions, choices)
 household_variables.append('bedperperson')
 conditions = [(tdf['rooms']==0),(tdf['rooms']>0)]
choices = [0, tdf['bedrooms']/tdf['rooms']]
tdf['perbedrooms']=np.select(conditions, choices)
 household_variables.append('perbedrooms')
 conditions = [(tdf['rooms']==0),(tdf['rooms']>0)]
choices = [0, tdf['tamhog']/tdf['rooms']]
tdf['rperperson']=np.select(conditions, choices)
 household_variables.append('rperperson')
 conditions = [(tdf['rooms']==0),(tdf['rooms']>0)]
choices = [0, tdf['bedrooms']/tdf['rooms']]
tdf['roomstobedrooms']=np.select(conditions, choices)
 household_variables.append('roomstobedrooms')
 conditions = [(tdf['hogar_adul']==0),(tdf['hogar_adul']>0)]
choices = [0, tdf['lv2a1']/tdf['hogar_adul']]
tdf['rentpadult']=np.select(conditions, choices)
 household_variables.append('rentpadult')
 conditions = [(tdf['hogar_adul']==0),(tdf['hogar_adul']>0)]
choices = [0, tdf['qmobilephone']/tdf['hogar_adul']]
tdf['phoneperadult']=np.select(conditions, choices)
 household_variables.append('phoneperadult')
 conditions = [(tdf['hhsize']==0),(tdf['hhsize']>0)]
choices = [0, tdf['qmobilephone']/tdf['hhsize']]
tdf['phoneperhhsize']=np.select(conditions, choices)
 household_variables.append('phoneperhhsize')
 conditions = [(tdf['rooms']==0),(tdf['rooms']>0)]
choices = [0, tdf['lv2a1']/tdf['rooms']]
tdf['rentproom']=np.select(conditions, choices)
 household_variables.append('rentproom')
 conditions = [(tdf['dependency']=='yes'),(tdf['dependency']=='no'),(tdf['dependency']!='yes')&(tdf['dependency']!='no')]
choices = [1,0,0]
tdf['dphhyesno']=np.select(conditions, choices)
 household_variables.append('dphhyesno')
 conditions = [(tdf['dependency']=='yes'),(tdf['dependency']=='no'),(tdf['dependency']!='yes')&(tdf['dependency']!='no')]
choices = [0,0,tdf['dependency']]
tdf['dpnumeric']=np.select(conditions, choices)
 household_variables.append('dpnumeric')
 tdf['femhead'] = 0
tdf.loc[(tdf['parentesco1']==1)&(tdf['female']==1),'femhead'] = 1
tdf['femhead'] = tdf['femhead'].groupby(tdf['idhogar'] ).transform('count')
 tdf['femhead'] = np.where(tdf['femhead']>=1, 1, 0)
 household_variables.append('femhead')
 tdf['hhheadeduindex'] = 0.0
tdf['hhheadeduindex'] = tdf[tdf['parentesco1']==1]['escolari'].groupby(tdf['idhogar'] ).transform('mean')
 tdf['hhheadeduindex'] =(tdf['hhheadeduindex'] / 15.0 +(7.2/18.0)) /2.0
tdf.loc[tdf['hhheadeduindex'].isna() ,'hhheadeduindex'] = 0
tdf['hhheadeduindex'] = tdf['hhheadeduindex'].groupby(tdf['idhogar'] ).transform('mean')
 tdf.loc[tdf['hhheadeduindex'].isna() ,'hhheadeduindex'] = 0
household_variables.append('hhheadeduindex')
 tdf['hhheadage'] =0
tdf['hhheadage'] = tdf[tdf['parentesco1']==1]['age'].groupby(tdf['idhogar'] ).transform('mean')
 tdf['hhheadage'] = tdf['hhheadage'].groupby(tdf['idhogar'] ).transform('mean')
 tdf.loc[tdf['hhheadage'].isna() ,'hhheadage'] = 0
household_variables.append('hhheadage')
 tdf['misshead'] = 0
tdf['misshead'] = tdf['parentesco1'].groupby(tdf['idhogar'] ).transform('sum')
 tdf['misshead'] = np.where(tdf['misshead']==0, 1, 0)
 household_variables.append('misshead')
 conditions = [(tdf['edjefe']=='yes'),(tdf['edjefe']=='no'),(tdf['edjefe']!='yes')&(tdf['edjefe']!='no')]
choices = [0,1,0]
tdf['edjefemiss']=np.select(conditions, choices)
 household_variables.append('edjefemiss')
 conditions = [(tdf['edjefe']=='yes'),(tdf['edjefe']=='no'),(tdf['edjefe']!='yes')&(tdf['edjefe']!='no')]
choices = [0,0,tdf['edjefe']]
tdf['edjefeval']=np.select(conditions, choices)
 household_variables.append('edjefeval')
 conditions = [(tdf['edjefa']=='yes'),(tdf['edjefa']=='no'),(tdf['edjefa']!='yes')&(tdf['edjefa']!='no')]
choices = [0,1,0]
tdf['edjefamiss']=np.select(conditions, choices)
 household_variables.append('edjefamiss')
 conditions = [(tdf['edjefa']=='yes'),(tdf['edjefa']=='no'),(tdf['edjefa']!='yes')&(tdf['edjefa']!='no')]
choices = [0,0,tdf['edjefa']]
tdf['edjefaval']=np.select(conditions, choices)
 household_variables.append('edjefaval')
 tdf['region'] = tdf['region'] = tdf[locations].idxmax(axis = 1)
 tdf['r4h1permale'] = tdf['r4h1'] / tdf['r4h3']
tdf.loc[tdf['r4h1permale'].isna() ,'r4h1permale'] =0
household_variables.append('r4h1permale')
 tdf['r4m1perfemale'] = tdf['r4m1'] / tdf['r4m3']
tdf.loc[tdf['r4m1perfemale'].isna() ,'r4m1perfemale'] =0
household_variables.append('r4m1perfemale')
 tdf['r4h1pertotal'] = tdf['r4h1'] / tdf['hhsize']
household_variables.append('r4h1pertotal')
 tdf['r4m1pertotal'] = tdf['r4m1'] / tdf['hhsize']
household_variables.append('r4m1pertotal')
 tdf['r4t1pertotal'] = tdf['r4t1'] / tdf['hhsize']
household_variables.append('r4t1pertotal')
 tdf['cleanwater'] = 0
tdf.loc[(tdf['abastaguafuera']==1)|(tdf['abastaguano']==1),'cleanwater'] = 1
household_variables.append('cleanwater')
 tdf['improvedsan'] = 0
tdf.loc[(tdf['sanitario2']==1)|(tdf['sanitario3']==1)|(tdf['sanitario5']==1),'improvedsan'] = 1
household_variables.append('improvedsan')
 tdf['electricity'] = 0
tdf.loc[(tdf['public']==1)|(tdf['planpri']==1)|(tdf['coopele']==1),'electricity'] = 1
household_variables.append('electricity')
 tdf['flooring'] = 0
tdf.loc[(tdf['pisoother']==1)|(tdf['pisonatur']==1)|(tdf['pisonotiene']==1)|(tdf['pisomadera']==1),'flooring'] = 1
household_variables.append('flooring')
 tdf['wall'] = 0
tdf.loc[(tdf['paredzocalo']==1)|(tdf['pareddes']==1)|(tdf['paredfibras']==1)|(tdf['paredother']==1)|(tdf['paredmad']==1),'wall'] = 1
household_variables.append('wall')
 tdf['roof'] = 0
tdf.loc[(tdf['techocane']==1)|(tdf['techootro']==1)|(tdf['cielorazo']==0),'roof'] = 1
household_variables.append('roof')
 tdf['cooking'] = 0
tdf.loc[(tdf['energcocinar4']==1)|(tdf['energcocinar1']==1),'cooking'] = 1
household_variables.append('cooking')
 tdf['rubbish'] = 0
tdf.loc[(tdf['elimbasu3']==1)|(tdf['elimbasu4']==1)|(tdf['elimbasu5']==1)|(tdf['elimbasu6']==1),'rubbish'] = 1
household_variables.append('rubbish')
 tdf['condition'] = 0
tdf.loc[(tdf['epared1']==1)&(tdf['etecho1']==1)&(tdf['eviv1']==1),'condition'] = 1
household_variables.append('condition')
 tdf['condition2'] = 0
tdf.loc[(tdf['epared2']==1)&(tdf['etecho2']==1)&(tdf['eviv2']==1),'condition2'] = 1
household_variables.append('condition2')
 tdf['condition3'] = 0
tdf.loc[(tdf['epared3']==1)&(tdf['etecho3']==1)&(tdf['eviv3']==1),'condition3'] = 1
household_variables.append('condition3')
 household_variables = list(set(household_variables))
train = train.loc[(train['dropoff_longitude'] > -75)&(train['dropoff_longitude'] < -73)]
train = train.loc[(train['dropoff_latitude'] > 40.5)&(train['dropoff_latitude'] < 41.5)]<filter>
df_regional = df.copy()
df_test_regional = df_test.copy()
df_test_regional["Target"]=0
df_test_regional["Target0"]=0
df_all = pd.concat([df_regional,df_test_regional],sort=False)
 df_all.index = range(0,len(df_all['idhogar']))
regional_variables = []
df_all['regionalMeanLogRent'] = 0.0
df_all['regionalMeanLogRent'] = df_all[df_all['lv2a1']>0][['lv2a1','idhogar','region']].groupby(['idhogar','region'] ).transform('mean' ).groupby(df_all['region'] ).transform('mean')
 df_all.loc[df_all['regionalMeanLogRent'].isna() ,'regionalMeanLogRent'] = 0
regional_variables.append('regionalMeanLogRent')
 df_all['rmedianage'] = df_all[['age','idhogar','region']].groupby(['region'] ).transform('median')
 regional_variables.append('rmedianage')
 df_all['reduindex'] = df_all[['eduindex','idhogar','region']].groupby(['idhogar','region'] ).transform('mean' ).groupby(df_all['region'] ).transform('mean')
 regional_variables.append('reduindex')
 df_all['rnhhlt14'] = df_all[['nhhlt14','idhogar','region']].groupby(['idhogar','region'] ).transform('mean' ).groupby(df_all['region'] ).transform('mean')
 regional_variables.append('rnhhlt14')
 df_all['rhogar_total'] = df_all[['hogar_total','idhogar','region']].groupby(['idhogar','region'] ).transform('mean' ).groupby(df_all['region'] ).transform('mean')
 regional_variables.append('rhogar_total')
 df_all['rbedrooms'] = df_all[['bedrooms','idhogar','region']].groupby(['idhogar','region'] ).transform('mean' ).groupby(df_all['region'] ).transform('mean')
 regional_variables.append('rbedrooms')
 df_all['urbanhouseholds']=df_all[['area1','idhogar','region']].groupby(['idhogar','region'] ).transform('mean' ).groupby(df_all['region'] ).transform('mean')
 regional_variables.append('urbanhouseholds')
 for fld in regional_variables:
tempPiv = pd.pivot_table(df_all, values=fld, index=['region'], aggfunc=np.mean)
 tempPiv.columns = [fld]
df = df.merge(tempPiv,on='region',how='left')
 del tempPiv
for fld in regional_variables:
tempPiv = pd.pivot_table(df_all, values=fld, index=['region'], aggfunc=np.mean)
 tempPiv.columns = [fld]
df_test = df_test.merge(tempPiv,on='region',how='left')
 del tempPiv
train = train.loc[(train['passenger_count'] >= 0)&(train['passenger_count'] <= 6)]<count_missing_values>
for tdf in dataframes:
print(tdf.columns[tdf.isna().any() ].tolist())
 for f in tdf.columns[tdf.isna().any() ].tolist() :
if f in regional_variables:
print(f)
 print(tdf.columns[tdf.isnull().any() ].tolist())
 for f in tdf.columns[tdf.isnull().any() ].tolist() :
if f in regional_variables:
print(f )
train.isnull().sum()<count_missing_values>
combo = individual_variables+household_variables+regional_variables
combo = list(set(combo))
index_individual_variables = list(range(0,len(individual_variables)))
 index_household_variables = list(range(len(individual_variables),len(individual_variables)+len(household_variables)))
 index_indiv_and_hh_variables = index_individual_variables+index_household_variables
train.isnull().sum()<count_duplicates>
keeplst = []
indvmodels = {}
for t in [1,2,3,4]:
params = {'n_estimators': 500,
'max_depth':None,
'min_samples_split':2,
'min_samples_leaf':90,
'max_features':'auto',
'loss':'deviance',
"learning_rate":.01,
"subsample":.5}
currentTarget = "Target_"+str(t)
 print(currentTarget)
 currentTarget_v = t
df_sub = df[df['parentesco1']==1][combo+[currentTarget,"idhogarnum",'targetweight']].copy()
X = df_sub[combo].values
y = df_sub[currentTarget].values.ravel()
weights = df_sub['targetweight'].values.ravel()
Y_strat = df_sub['idhogarnum'].values
gss = GroupShuffleSplit(test_size=.1, n_splits=1,random_state=314)
 for train_index, test_index in gss.split(X, y, groups=Y_strat):
break
X_train, X_test = X[train_index],X[test_index]
y_train, y_test = y[train_index],y[test_index]
w_train = weights[train_index]
strat_train = Y_strat[train_index]
print(X_train.shape)
 print(X_test.shape)
 reg = GradientBoostingClassifier(**params)
 reg.fit(X_train,y_train,sample_weight=w_train)
 indvmodels[t] = reg
y_pred = reg.predict(X_test)
 print(f1_score(y_test, y_pred, average='macro'))
print(classification_report(y_test, y_pred))
fi = list(zip(range(0,len(reg.feature_importances_)) ,reg.feature_importances_))
fi.sort(key=lambda x: x[1])
 vals = [x[1] for x in fi]
for i,important in fi:
if important >=np.percentile(reg.feature_importances_, 75):
keeplst.append(combo[i])
 print("%s: %s"%(combo[i],important))
probabilities = pd.DataFrame(reg.predict_proba(X),columns=["P0","P1"])
 df[currentTarget+"_p"]=0.1
df.loc[df['parentesco1']==1,currentTarget+"_p"]= probabilities["P1"].values
print(list(set(keeplst)) )
train.duplicated().sum()<count_duplicates>
df['pscore'] = 0.0
for t in [1,2,3,4]:
df.loc[df['Target']==t,'pscore'] = df.loc[df['Target']==t]["Target_"+str(t)+'_p']
print(df[df['Target']==1].head())
 df['indsampweight']= 100.0*df['pscore']
df['targetpscore'] = df['targetweight']*df['pscore']
df['targetpscore'] = df['targetpscore'] / df['targetpscore'].sum()
print(df[df['Target']==1].head())
 y_pred = KMeans(n_clusters=4, random_state=314 ).fit_predict(df['pscore'].values.reshape(-1, 1))
df['strata'] = y_pred
df[['Target','pscore','strata']]
df[df['strata']==1][['Target','pscore','strata']]
pd.pivot_table(df,values='pscore',index='strata',aggfunc="mean" )
train = train.drop_duplicates()
train.duplicated().sum()<drop_column>
keeplst = list(set(keeplst))
df_sub = df[combo+["Target0","idhogarnum",'strata','indsampweight','targetweight','targetpscore']].copy()
X = df_sub[combo].values
y = df_sub["Target0"].values.ravel()
Y_strat = df_sub["idhogarnum"]
weights_target = df_sub['targetweight'].values.ravel()
weights_all = df_sub['indsampweight'].values.ravel()
weights_tp = df_sub['targetpscore'].values.ravel()
gss = GroupShuffleSplit(test_size=.1, n_splits=1,random_state=314)
 gss = ShuffleSplit(n_splits=1, test_size=.1, random_state=314)
 for train_index, test_index in gss.split(X, y):
break
X_train, X_test = X[train_index],X[test_index]
y_train, y_test = y[train_index],y[test_index]
wtar_train = weights_target[train_index]
wall_train = weights_all[train_index]
wtp_train = weights_tp[train_index]
strat_train = Y_strat[train_index]
print(X_train.shape)
 print(X_test.shape)
 params = {'n_estimators': 500,
'max_depth': None,
'min_samples_split':10,
'min_samples_leaf':90,
'max_features':'auto',
'loss':'deviance',
"learning_rate":.01,
"subsample":.5}
reg = GradientBoostingClassifier(**params)
 reg.fit(X_train,y_train,sample_weight=wtp_train)
 y_pred = reg.predict(X_test)
 print(f1_score(y_test, y_pred, average='macro'))
print(classification_report(y_test, y_pred))
train.drop(["store_and_fwd_flag"], axis=1, inplace=True)
 test.drop(["store_and_fwd_flag"], axis=1, inplace=True )<categorify>
X_fin = df_test[combo].values
df_test['TargetP'] = reg.predict(X_fin)+1
print(df_test.TargetP)
 df_output = pd.DataFrame({"Id":df_test['Id'],"Target":df_test['TargetP']})
 df_output.to_csv("outputTargetPropensity.csv",index=False )
<compute_test_metric><EOS> 
X_fin = df_test[combo].values
y_indiv = np.zeros([X_fin.shape[0], 4], dtype=float)
 for t in [1,2,3,4]:
reg = indvmodels[t]
df_prob = pd.DataFrame(reg.predict_proba(X_fin),columns=['B0','B1'])
 y_indiv[:, t-1] = df_prob['B1'].values
del df_prob
y_pred= np.zeros(X_fin.shape[0])
 for i in range(X_fin.shape[0]):
y_pred[i]=np.argmax(y_indiv[i])
 y_pred = np.array(y_pred,dtype=int)
 df_test['TargetI'] = y_pred+1
print(df_test.TargetI)
 df_output = pd.DataFrame({"Id":df_test['Id'],"Target":df_test['TargetI']})
 df_output.to_csv("outputTargetIndiv.csv",index=False )