Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<merge>
warnings.filterwarnings("ignore")
olid_test = olid_test.set_index('id' ).join(olid_test_label.set_index('id'))<data_type_conversions>
df = pd.read_csv('.. /input/sample_submission.csv')
 for i in np.arange(1,5):
df['Target'] = i
df.to_csv('sample_{}.csv'.format(i), index = False )
olid_test['toxic'] =(olid_test.label == 'OFF' ).astype(int ).values<split>
train_df = pd.read_csv('.. /input/train.csv')
 train_scored = train_df.query('parentesco1 == 1')
 train_dist = train_scored['Target'].value_counts(normalize = True ).tolist()
train_dist = [train_dist[3],train_dist[1],train_dist[2],train_dist[0]]
print('Train distributions are', train_dist)
 difference = target_dist - train_dist
print('The difference in the test and target distributions is', difference )
olid_train, olid_valid = train_test_split(olid, test_size=3240, random_state = 2020)
 olid_train_1k = olid_train[0:1000]
olid_train_2k = olid_train[0:2000]
olid_train_5k = olid_train[0:5000] 
<categorify>
train_set = pd.read_csv('.. /input/train.csv')
 test_set = pd.read_csv('.. /input/test.csv')
 test_set.loc[test_set['rez_esc'] == 99.0 , 'rez_esc'] = 5
def repalce_v18q1(x):
if x['v18q'] == 0:
return x['v18q']
else:
return x['v18q1']
train_set['v18q1'] = train_set.apply(lambda x : repalce_v18q1(x),axis=1)
 test_set['v18q1'] = test_set.apply(lambda x : repalce_v18q1(x),axis=1)
 train_set['v2a1'] = train_set['v2a1'].fillna(value=train_set['tipovivi3'])
 test_set['v2a1'] = test_set['v2a1'].fillna(value=test_set['tipovivi3'])
 cols = ['edjefe', 'edjefa', 'dependency']
train_set[cols] = train_set[cols].replace({'no': 0, 'yes':1} ).astype(float)
 test_set[cols] = test_set[cols].replace({'no': 0, 'yes':1} ).astype(float)
 train_set['roof_waste_material'] = np.nan
test_set['roof_waste_material'] = np.nan
train_set['electricity_other'] = np.nan
test_set['electricity_other'] = np.nan
def fill_roof_exception(x):
if(x['techozinc'] == 0)and(x['techoentrepiso'] == 0)and(x['techocane'] == 0)and(x['techootro'] == 0):
return 1
else:
return 0
def fill_no_electricity(x):
if(x['public'] == 0)and(x['planpri'] == 0)and(x['noelec'] == 0)and(x['coopele'] == 0):
return 1
else:
return 0
train_set['roof_waste_material'] = train_set.apply(lambda x : fill_roof_exception(x),axis=1)
 test_set['roof_waste_material'] = test_set.apply(lambda x : fill_roof_exception(x),axis=1)
 train_set['electricity_other'] = train_set.apply(lambda x : fill_no_electricity(x),axis=1)
 test_set['electricity_other'] = test_set.apply(lambda x : fill_no_electricity(x),axis=1)
 def owner_is_adult(x):
if x['age'] <= 18:
return 0
else:
return 1
train_set['head_less_18'] = train_set.apply(lambda x : owner_is_adult(x),axis=1)
 test_set['head_less_18'] = test_set.apply(lambda x : owner_is_adult(x),axis=1)
 train_set['adult'] = train_set['hogar_adul'] - train_set['hogar_mayor']
train_set['dependency_count'] = train_set['hogar_nin'] + train_set['hogar_mayor']
train_set['dependency'] = train_set['dependency_count'] / train_set['adult']
train_set['child_percent'] = train_set['hogar_nin']/train_set['hogar_total']
train_set['elder_percent'] = train_set['hogar_mayor']/train_set['hogar_total']
train_set['adult_percent'] = train_set['hogar_adul']/train_set['hogar_total']
test_set['adult'] = test_set['hogar_adul'] - test_set['hogar_mayor']
test_set['dependency_count'] = test_set['hogar_nin'] + test_set['hogar_mayor']
test_set['dependency'] = test_set['dependency_count'] / test_set['adult']
test_set['child_percent'] = test_set['hogar_nin']/test_set['hogar_total']
test_set['elder_percent'] = test_set['hogar_mayor']/test_set['hogar_total']
test_set['adult_percent'] = test_set['hogar_adul']/test_set['hogar_total']
train_set['rent_per_adult'] = train_set['v2a1']/(train_set['hogar_adul']+0.1)
 train_set['rent_per_person'] = train_set['v2a1']/train_set['hhsize']
test_set['rent_per_adult'] = test_set['v2a1']/(test_set['hogar_adul']+0.1)
 test_set['rent_per_person'] = test_set['v2a1']/test_set['hhsize']
train_set['overcrowding_room_and_bedroom'] =(train_set['hacdor'] + train_set['hacapo'])/2
test_set['overcrowding_room_and_bedroom'] =(test_set['hacdor'] + test_set['hacapo'])/2
train_set['no_appliances'] = train_set['refrig'] + train_set['computer'] + train_set['television']
test_set['no_appliances'] = test_set['refrig'] + test_set['computer'] + test_set['television']
train_set['r4h1_percent_in_male'] = train_set['r4h1'] / train_set['r4h3']
train_set['r4m1_percent_in_female'] = train_set['r4m1'] / train_set['r4m3']
train_set['r4h1_percent_in_total'] = train_set['r4h1'] / train_set['hhsize']
train_set['r4m1_percent_in_total'] = train_set['r4m1'] / train_set['hhsize']
train_set['r4t1_percent_in_total'] = train_set['r4t1'] / train_set['hhsize']
test_set['r4h1_percent_in_male'] = test_set['r4h1'] / test_set['r4h3']
test_set['r4m1_percent_in_female'] = test_set['r4m1'] / test_set['r4m3']
test_set['r4h1_percent_in_total'] = test_set['r4h1'] / test_set['hhsize']
test_set['r4m1_percent_in_total'] = test_set['r4m1'] / test_set['hhsize']
test_set['r4t1_percent_in_total'] = test_set['r4t1'] / test_set['hhsize']
train_set['rent_per_room'] = train_set['v2a1']/train_set['rooms']
train_set['bedroom_per_room'] = train_set['bedrooms']/train_set['rooms']
train_set['elder_per_room'] = train_set['hogar_mayor']/train_set['rooms']
train_set['adults_per_room'] = train_set['adult']/train_set['rooms']
train_set['child_per_room'] = train_set['hogar_nin']/train_set['rooms']
train_set['male_per_room'] = train_set['r4h3']/train_set['rooms']
train_set['female_per_room'] = train_set['r4m3']/train_set['rooms']
train_set['room_per_person_household'] = train_set['hhsize']/train_set['rooms']
test_set['rent_per_room'] = test_set['v2a1']/test_set['rooms']
test_set['bedroom_per_room'] = test_set['bedrooms']/test_set['rooms']
test_set['elder_per_room'] = test_set['hogar_mayor']/test_set['rooms']
test_set['adults_per_room'] = test_set['adult']/test_set['rooms']
test_set['child_per_room'] = test_set['hogar_nin']/test_set['rooms']
test_set['male_per_room'] = test_set['r4h3']/test_set['rooms']
test_set['female_per_room'] = test_set['r4m3']/test_set['rooms']
test_set['room_per_person_household'] = test_set['hhsize']/test_set['rooms']
train_set['rent_per_bedroom'] = train_set['v2a1']/train_set['bedrooms']
train_set['edler_per_bedroom'] = train_set['hogar_mayor']/train_set['bedrooms']
train_set['adults_per_bedroom'] = train_set['adult']/train_set['bedrooms']
train_set['child_per_bedroom'] = train_set['hogar_nin']/train_set['bedrooms']
train_set['male_per_bedroom'] = train_set['r4h3']/train_set['bedrooms']
train_set['female_per_bedroom'] = train_set['r4m3']/train_set['bedrooms']
train_set['bedrooms_per_person_household'] = train_set['hhsize']/train_set['bedrooms']
test_set['rent_per_bedroom'] = test_set['v2a1']/test_set['bedrooms']
test_set['edler_per_bedroom'] = test_set['hogar_mayor']/test_set['bedrooms']
test_set['adults_per_bedroom'] = test_set['adult']/test_set['bedrooms']
test_set['child_per_bedroom'] = test_set['hogar_nin']/test_set['bedrooms']
test_set['male_per_bedroom'] = test_set['r4h3']/test_set['bedrooms']
test_set['female_per_bedroom'] = test_set['r4m3']/test_set['bedrooms']
test_set['bedrooms_per_person_household'] = test_set['hhsize']/test_set['bedrooms']
train_set['tablet_per_person_household'] = train_set['v18q1']/train_set['hhsize']
train_set['phone_per_person_household'] = train_set['qmobilephone']/train_set['hhsize']
test_set['tablet_per_person_household'] = test_set['v18q1']/test_set['hhsize']
test_set['phone_per_person_household'] = test_set['qmobilephone']/test_set['hhsize']
train_set['age_12_19'] = train_set['hogar_nin'] - train_set['r4t1']
test_set['age_12_19'] = test_set['hogar_nin'] - test_set['r4t1']
train_set['escolari_age'] = train_set['escolari']/train_set['age']
test_set['escolari_age'] = test_set['escolari']/test_set['age']
train_set['rez_esc_escolari'] = train_set['rez_esc']/(train_set['escolari']+0.1)
 train_set['rez_esc_r4t1'] = train_set['rez_esc']/(train_set['r4t1']+0.1)
 train_set['rez_esc_r4t2'] = train_set['rez_esc']/train_set['r4t2']
train_set['rez_esc_r4t3'] = train_set['rez_esc']/train_set['r4t3']
train_set['rez_esc_age'] = train_set['rez_esc']/train_set['age']
test_set['rez_esc_escolari'] = test_set['rez_esc']/(test_set['escolari']+0.1)
 test_set['rez_esc_r4t1'] = test_set['rez_esc']/(test_set['r4t1']+0.1)
 test_set['rez_esc_r4t2'] = test_set['rez_esc']/test_set['r4t2']
test_set['rez_esc_r4t3'] = test_set['rez_esc']/test_set['r4t3']
test_set['rez_esc_age'] = test_set['rez_esc']/test_set['age']
train_set['dependency'] = train_set['dependency'].replace({np.inf: 0})
 test_set['dependency'] = test_set['dependency'].replace({np.inf: 0})
 df_train = pd.DataFrame()
df_test = pd.DataFrame()
aggr_mean_list = ['rez_esc', 'dis', 'male', 'female', 'estadocivil1', 'estadocivil2', 'estadocivil3', 'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7', 'parentesco2',
'parentesco3', 'parentesco4', 'parentesco5', 'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10', 'parentesco11', 'parentesco12',
'instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9',]
other_list = ['escolari', 'age', 'escolari_age']
for item in aggr_mean_list:
group_train_mean = train_set[item].groupby(train_set['idhogar'] ).mean()
group_test_mean = test_set[item].groupby(test_set['idhogar'] ).mean()
new_col = item + '_aggr_mean'
df_train[new_col] = group_train_mean
df_test[new_col] = group_test_mean
for item in other_list:
for function in ['mean','std','min','max','sum']:
group_train = train_set[item].groupby(train_set['idhogar'] ).agg(function)
 group_test = test_set[item].groupby(test_set['idhogar'] ).agg(function)
 new_col = item + '_' + function
df_train[new_col] = group_train
df_test[new_col] = group_test
df_test = df_test.reset_index()
df_train = df_train.reset_index()
train_agg = pd.merge(train_set, df_train, on='idhogar')
 test = pd.merge(test_set, df_test, on='idhogar')
 train_agg.fillna(value=0, inplace=True)
 test.fillna(value=0, inplace=True)
 train = train_agg.query('parentesco1==1')
 submission = test[['Id']]
test = test.query('parentesco1==1')
 pred_train_dist = test[['Id']]
pred_test_dist = test[['Id']]
train.drop(columns=['idhogar','Id', 'tamhog', 'agesq', 'hogar_adul', 'SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe', 'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned'], inplace=True)
 test.drop(columns=['idhogar','Id', 'tamhog', 'agesq', 'hogar_adul', 'SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe', 'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned'], inplace=True)
 print('Train shape:',train.shape)
 print('Test shape:', test.shape)
 y = train['Target']
train.drop(columns=['Target'], inplace=True)
 scaler = MinMaxScaler()
train = scaler.fit_transform(train)
 test = scaler.transform(test)
 def classifyrank(data, cutoff):
data = rankdata(data)
 length = data.size
clas = []
for j in data:
i = j/length
if i < cutoff[0]:
clas.append(1)
 elif i < cutoff[0]+cutoff[1]:
clas.append(2)
 elif i < cutoff[0]+cutoff[1]+cutoff[2]:
clas.append(3)
 else:
clas.append(4)
 return clas
def fast_encode_xlm(texts, tokenizer, chunk_size=256, maxlen=512):

 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.batch_encode_plus(text_chunk, pad_to_max_length = True, max_length = maxlen)
 all_ids.extend(np.array(encs.input_ids))
return np.array(all_ids )<prepare_x_and_y>
clf = Ridge(alpha = 5)
 kfold = 5
kf = StratifiedKFold(n_splits=kfold, shuffle=True, random_state = 9)
 predicts_result = []
ave_score = 0
for train_index, test_index in kf.split(train, y):
X_train, X_val = train[train_index], train[test_index]
y_train, y_val = y.iloc[train_index], y.iloc[test_index]
clf.fit(X_train, y_train)
 predicts_result.append(clf.predict(test))
ave_score += f1_score(y_val, classifyrank(clf.predict(X_val), train_dist), average = 'macro')/kfold
print('Cross Validation Macro F1 score:',ave_score )
<categorify><EOS> 
predicts_result = rankdata(np.array(predicts_result ).mean(axis = 0))
target_train_dist = classifyrank(predicts_result, train_dist)
 target_test_dist = classifyrank(predicts_result, target_dist)
 pred_train_dist['Target'] = target_train_dist
pred_test_dist['Target'] = target_test_dist
print('Using Train Distribution:')
 print(pred_train_dist['Target'].value_counts())
 print('
Using Test Distribution:')
 print(pred_test_dist['Target'].value_counts())
 train_submission = submission.copy()
train_submission = train_submission.merge(pred_train_dist[['Id', 'Target']], on = 'Id', how = 'left')
 train_submission = train_submission.fillna(4)
 train_submission['Target'] = train_submission['Target'].astype(int)
 train_submission.to_csv('train_dist.csv', index = False)
 test_submission = submission.copy()
test_submission = test_submission.merge(pred_test_dist[['Id', 'Target']], on = 'Id', how = 'left')
 test_submission = test_submission.fillna(4)
 test_submission['Target'] = test_submission['Target'].astype(int)
 test_submission.to_csv('test_dist.csv', index = False )