Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<predict_on_test>
%matplotlib inline
%reload_ext autoreload
%autoreload 2
np.set_printoptions(threshold=50, edgeitems=20)
 pd.set_option('display.max_rows', 500)
 pd.set_option('display.max_columns', 500)
cnn_preds = cnn_trainer.predict(make_cnn_inputs(stage3_preds, stage_1_2_target_cols))<compute_train_metric>
PATH = '.. /input/costa-rican-household-poverty-prediction/'
train_df = pd.read_csv(PATH+'train.csv')
 test_df = pd.read_csv(PATH+'test.csv' )
def get_score(weights, train_idx, oofs, labels):
blend = np.zeros_like(oofs[0][train_idx, :])
 for oof, weight in zip(oofs[:-1], weights):
blend += weight * oof[train_idx, :]
blend +=(1 - np.sum(weights)) * oofs[-1][train_idx, :]
return logloss_for_multilabel(labels[train_idx, :], blend)
 def get_best_weights(oofs, labels):
weight_list = []
weights = np.array([1/len(oofs)for x in range(len(oofs)- 1)])
 for n_splits in tqdm([5, 6]):
for i in range(2):
kf = KFold(n_splits=n_splits, random_state=i, shuffle=True)
 for fold,(train_idx, valid_idx)in enumerate(kf.split(X=oofs[0])) :
res = minimize(get_score, weights, args=(train_idx, oofs, labels), method="Nelder-Mead", tol=1e-6)
 logger.info(f"i: {i} fold: {fold} res.x: {res.x}")
 weight_list.append(res.x)
 mean_weight = np.mean(weight_list, axis=0)
 print(f"optimized weight: {mean_weight}")
 return mean_weight
best_weights = get_best_weights(stage3_oofs, train_features_df[stage_1_2_target_cols].values )<categorify>
cat_vars = ['hacdor',
'hacapo',
'v14a',
'refrig',
'v18q',
'paredblolad',
'paredzocalo',
'paredpreb',
'pareddes',
'paredmad',
'paredzinc',
'paredfibras',
'paredother',
'pisomoscer',
'pisocemento',
'pisoother',
'pisonatur',
'pisonotiene',
'pisomadera',
'techozinc',
'techoentrepiso',
'techocane',
'techootro',
'cielorazo',
'abastaguadentro',
'abastaguafuera',
'abastaguano',
'public',
'planpri',
'noelec',
'coopele',
'sanitario1',
'sanitario2',
'sanitario3',
'sanitario5',
'sanitario6',
'energcocinar1',
'energcocinar2',
'energcocinar3',
'energcocinar4',
'elimbasu1',
'elimbasu2',
'elimbasu3',
'elimbasu4',
'elimbasu5',
'elimbasu6',
'epared1',
'epared2',
'epared3',
'etecho1',
'etecho2',
'etecho3',
'eviv1',
'eviv2',
'eviv3',
'dis',
'male',
'female',
'estadocivil1',
'estadocivil2',
'estadocivil3',
'estadocivil4',
'estadocivil5',
'estadocivil6',
'estadocivil7',
'parentesco1',
'parentesco2',
'parentesco3',
'parentesco4',
'parentesco5',
'parentesco6',
'parentesco7',
'parentesco8',
'parentesco9',
'parentesco10',
'parentesco11',
'parentesco12',
'dependency',
'edjefe',
'edjefa',
'instlevel1',
'instlevel2',
'instlevel3',
'instlevel4',
'instlevel5',
'instlevel6',
'instlevel7',
'instlevel8',
'instlevel9',
'tipovivi1',
'tipovivi2',
'tipovivi3',
'tipovivi4',
'tipovivi5',
'computer',
'television',
'mobilephone',
'lugar1',
'lugar2',
'lugar3',
'lugar4',
'lugar5',
'lugar6',
'area1',
'area2',
'idhogar']
contin_vars = ['v2a1',
'rooms',
'v18q1',
'r4h1',
'r4h2',
'r4h3',
'r4m1',
'r4m2',
'r4m3',
'r4t1',
'r4t2',
'r4t3',
'tamhog',
'escolari',
'rez_esc',
'hhsize',
'hogar_nin',
'hogar_adul',
'hogar_mayor',
'hogar_total',
'meaneduc',
'bedrooms',
'overcrowding',
'qmobilephone',
'age',
'SQBescolari',
'SQBage',
'SQBhogar_total',
'SQBedjefe',
'SQBhogar_nin',
'SQBovercrowding',
'SQBdependency',
'SQBmeaned',
'agesq']
objective= ['Target']
ids = ['Id']
best_weights = np.insert(best_weights, len(best_weights), 1 - np.sum(best_weights))
oof_preds = np.stack(stage3_oofs ).transpose(1, 2, 0 ).dot(best_weights)
 print(logloss_for_multilabel(train_features_df[stage_1_2_target_cols].values, oof_preds))
blend_preds = np.stack(stage3_preds ).transpose(1, 2, 0 ).dot(best_weights )<feature_engineering>
for v in cat_vars: train_df[v] = train_df[v].astype('category' ).cat.as_ordered()
apply_cats(test_df, train_df )
final_blend_oofs = oof_preds * 0.7 + cnn_trainer.result.oof_preds * 0.3
test_features_df[stage_1_2_target_cols] = blend_preds * 0.7 + cnn_preds * 0.3<compute_test_metric>
for v in contin_vars:
train_df[v] = train_df[v].fillna(0 ).astype('float32')
 test_df[v] = test_df[v].fillna(0 ).astype('float32' )
print(logloss_for_multilabel(train_features_df[stage_1_2_target_cols].values, final_blend_oofs))<define_variables>
for dataframe in(train_df, test_df):
dataframe['Rent_per_individual'] = dataframe['v2a1']/dataframe['r4t3']
dataframe['Rent_per_child'] = dataframe['v2a1']/dataframe['r4t1']
dataframe['Rent_per_over_65'] = dataframe['v2a1']/dataframe['r4t3']
dataframe['Rent_per_room'] = dataframe['v2a1']/dataframe['rooms']
dataframe['Rent_per_bedrooms'] = dataframe['v2a1']/dataframe['bedrooms']
dataframe['Proportion_under_12'] = dataframe['r4t1']/dataframe['r4t3']
dataframe['Proportion_under_12_male'] = dataframe['r4h1']/dataframe['r4t3']
dataframe['Proportion_under_12_female'] = dataframe['r4m1']/dataframe['r4t3']
dataframe['Proportion_male'] = dataframe['r4h3']/dataframe['r4t3']
dataframe['Proportion_female'] = dataframe['r4m3']/dataframe['r4t3']
dataframe['Rooms_per_individual'] = dataframe['rooms']/dataframe['r4t3']
dataframe['Rooms_per_child'] = dataframe['rooms']/dataframe['r4t1']
dataframe['Tablets_per_individual'] = dataframe['v18q1']/dataframe['r4t3']
dataframe['Tablets_per_child'] = dataframe['v18q1']/dataframe['r4t1']
dataframe['Years_schooling_per_individual'] = dataframe['escolari']/dataframe['r4t3']
dataframe['Years_schooling_per_adult'] = dataframe['escolari']/(dataframe['r4t3']-dataframe['r4t1'])
 dataframe['Years_schooling_per_child'] = dataframe['escolari']/dataframe['r4t1']
dataframe['Proportion_under_19'] = dataframe['hogar_nin']/dataframe['r4t3']
dataframe['Proportion_over_19'] = dataframe['hogar_adul']/dataframe['r4t3']
dataframe['Proportion_under_65'] =(dataframe['hogar_total']-dataframe['hogar_mayor'])/dataframe['r4t3']
dataframe['Proportion_over_65'] = dataframe['hogar_mayor']/dataframe['r4t3']
dataframe['Bedrooms_per_individual'] = dataframe['bedrooms']/dataframe['r4t3']
dataframe['Bedrooms_per_child'] = dataframe['bedrooms']/dataframe['r4t1']
dataframe['Bedrooms_per_over_65'] = dataframe['bedrooms']/dataframe['r4t3']
dataframe['Extreme_conditions_flag'] =(dataframe['abastaguano'] & dataframe['noelec'] & dataframe['sanitario1'] & dataframe['energcocinar1'])
 dataframe['bedrooms_to_rooms'] = dataframe['bedrooms']/dataframe['rooms']
dataframe['tamhog_to_rooms'] = dataframe['tamhog']/dataframe['rooms']
dataframe['tamhog_to_bedrooms'] = dataframe['tamhog']/dataframe['bedrooms']
dataframe['r4t3_to_tamhog'] = dataframe['r4t3']/dataframe['tamhog']
dataframe['hhsize_to_rooms'] = dataframe['hhsize']/dataframe['rooms']
dataframe['hhsize_to_bedrooms'] = dataframe['hhsize']/dataframe['bedrooms']
dataframe['rent_to_hhsize'] = dataframe['v2a1']/dataframe['hhsize']
dataframe['qmobilephone_to_r4t3'] = dataframe['qmobilephone']/dataframe['r4t3']
dataframe['qmobilephone_to_v18q1'] = dataframe['qmobilephone']/dataframe['v18q1']
class PostProcessor(object):
def __init__(self, original_oof, original_pred_df):
self.original_oof = original_oof
self.original_pred_df = original_pred_df
if "_dh" in globals().keys() :
if globals() ["_dh"][0] == "/kaggle/working":
self.working_place = "kaggle_kernel"
self.base_dir = '.. /input'
self.csv_dir = ".. /input/lish-moa/"
else:
self.working_place = "local_lab"
self.base_dir = '.. /input'
self.csv_dir = ".. /input/lish-moa/"
else:
self.working_place = "local_py"
self.base_dir = '.. /input'
self.csv_dir = ".. /input/lish-moa/"
self.int2drug_id = {1: '87d714366', 2: '9f80f3f77', 3: '8b87a7a83', 4: '5628cb3ee',
5: 'd08af5d4b', 6: '292ab2c28', 7: 'd50f18348', 8: 'd1b47f29d'}
test, targets_with_drug_ids, target_cols = self.load_csv()
self.test = test
self.targets_with_drug_ids = targets_with_drug_ids
self.target_cols = target_cols
def load_csv(self):
test = pd.read_csv(os.path.join(self.csv_dir + "test_features.csv"))
train = pd.read_csv(os.path.join(self.csv_dir + "train_features.csv"))
train = train[train['cp_type'] == 'trt_cp'].reset_index(drop=True)
 targets = pd.read_csv(os.path.join(self.csv_dir + "train_targets_scored.csv"))
target_cols = [x for x in targets.columns if x not in ['sig_id', 'drug_id']]
drugs = pd.read_csv(os.path.join(self.csv_dir + "train_drug.csv"))
targets_with_drug_ids = train.merge(drugs, on='sig_id', how='left')
 targets_with_drug_ids = targets_with_drug_ids.merge(targets, on='sig_id', how='left')
 return test, targets_with_drug_ids, target_cols
def loss(self, target_arr, oof_arr):
y = torch.tensor(target_arr, dtype=float)
 p = torch.tensor(oof_arr, dtype=float)
 p = torch.clamp(p, 1e-15, 1-(1e-15))
competition_metric = nn.BCELoss()(p, y ).item()
return np.mean(competition_metric)
 def get_feature(self, df):
df["cp_dose"] =(df["cp_dose"] == "D1" ).astype(int)
 df["cp_time"] = df["cp_time"] / 72
features_g = ["g-" + str(i)for i in range(772)]
features_c = ["c-" + str(i)for i in range(100)]
df["g_sum"] = df[features_g].sum(axis=1)
 df["g_mean"] = df[features_g].mean(axis=1)
 df["g_median"] = df[features_g].median(axis=1)
 df["g_std"] = df[features_g].std(axis=1)
 df["g_kurt"] = df[features_g].kurtosis(axis=1)
 df["g_skew"] = df[features_g].skew(axis=1)
 df["c_sum"] = df[features_c].sum(axis=1)
 df["c_mean"] = df[features_c].mean(axis=1)
 df["c_std"] = df[features_c].std(axis=1)
 df["c_median"] = df[features_c].median(axis=1)
 df["c_kurt"] = df[features_c].kurtosis(axis=1)
 df["c_skew"] = df[features_c].skew(axis=1)
 df["gc_sum"] = df[features_g + features_c].sum(axis=1)
 df["gc_mean"] = df[features_g + features_c].mean(axis=1)
 df["gc_std"] = df[features_g + features_c].std(axis=1)
 df["gc_kurt"] = df[features_g + features_c].kurtosis(axis=1)
 df["gc_skew"] = df[features_g + features_c].skew(axis=1)
 df["gc_median"] = df[features_g + features_c].median(axis=1)
 return df
def fit(self):
with open(f"{self.base_dir}/drug-prediction-models/drug_id_oof.pkl", "rb")as f:
drug_id_prediction = pickle.load(f)
 oof_df = pd.DataFrame(self.original_oof, columns=self.target_cols)
 best_score = 1000
best_th = 0
for threth in [0.5, 0.9, 0.95, 0.96, 0.97, 0.98, 0.99]:
drug_id_prediction_hard = self.prediction_to_int(drug_id_prediction, threthold=threth)
 oof_df_update = self.update_prediction(oof_df, drug_id_prediction_hard)
 oof_arr = self.targets_with_drug_ids[self.target_cols].values
pred_arr = oof_df_update[self.target_cols].values
tmp_score = self.loss(oof_arr, pred_arr)
 print(threth, tmp_score)
 if tmp_score < best_score:
best_score = tmp_score
best_th = threth
return best_th, best_score
def prediction_to_int(self, drug_id_prediction, threthold=None):
if threthold is None:
threthold = 0.95
drug_id_prediction[drug_id_prediction < threthold] = 0
drug_id_prediction_hard = np.argmax(drug_id_prediction, axis=1)
 return drug_id_prediction_hard
def update_prediction(self, pred_df, drug_id_prediction_hard):
_pred_df = pred_df.copy()
for pred_int, drug_id in self.int2drug_id.items() :
idx =(drug_id_prediction_hard == pred_int)
 targets_dict = self.targets_with_drug_ids[self.targets_with_drug_ids['drug_id'] == drug_id][self.target_cols].head(1 ).reset_index(drop=True ).T.to_dict() [0]
for k, v in targets_dict.items() :
_pred_df.loc[idx, k] = v
return _pred_df
def transform(self, threthold=None):
with open(f"{self.base_dir}/drug-prediction-models/drug_id_models.pkl", "rb")as f:
models = pickle.load(f)
 test = self.get_feature(self.test)
 predictors = [x for x in test.columns if x not in ["sig_id", "cp_type", "drug_id"]]
drug_id_prediction = []
for model in models:
drug_id_prediction.append(model.predict(test[predictors]))
drug_id_prediction = np.mean(drug_id_prediction, axis=0)
 drug_id_prediction_hard = self.prediction_to_int(drug_id_prediction, threthold=None)
 pred_df = self.original_pred_df.copy()
pred_df = self.update_prediction(pred_df, drug_id_prediction_hard)
 return pred_df<categorify>
new_features = ['Rent_per_individual',
'Rent_per_child',
'Rent_per_over_65',
'Rent_per_room',
'Rent_per_bedrooms',
'Proportion_under_12',
'Proportion_under_12_male',
'Proportion_under_12_female',
'Proportion_male',
'Proportion_female',
'Rooms_per_individual',
'Rooms_per_child',
'Tablets_per_individual',
'Tablets_per_child',
'Years_schooling_per_individual',
'Years_schooling_per_adult',
'Years_schooling_per_child',
'Proportion_under_19',
'Proportion_over_19',
'Proportion_under_65',
'Proportion_over_65',
'Bedrooms_per_individual',
'Bedrooms_per_child',
'Bedrooms_per_over_65',
'Extreme_conditions_flag',
'bedrooms_to_rooms',
'tamhog_to_rooms',
'tamhog_to_bedrooms',
'r4t3_to_tamhog',
'hhsize_to_rooms',
'hhsize_to_bedrooms',
'rent_to_hhsize',
'qmobilephone_to_r4t3',
'qmobilephone_to_v18q1']
for v in new_features:
train_df[v] = train_df[v].fillna(0 ).astype('float32')
 test_df[v] = test_df[v].fillna(0 ).astype('float32' )
pp = PostProcessor(original_oof=final_blend_oofs,
original_pred_df=test_features_df[stage_1_2_target_cols])
 best_th, best_score = pp.fit()
print('best is', best_th, best_score)
 pp_prediction = pp.transform(best_th)
 test_features_df[stage_1_2_target_cols] = pp_prediction.values<save_to_csv>
train_df = train_df.set_index('Id')
 train_df = train_df[cat_vars+contin_vars+objective]
train_X, train_y, nas, mapper = proc_df(train_df, 'Target', do_scale=True )
test_features_df.loc[test_features_df['cp_type'] != 'trt_cp', stage_1_2_target_cols] = 0
sub = sample_submission_df.drop(columns=stage_1_2_target_cols ).merge(test_features_df[['sig_id']+stage_1_2_target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False)
 sub.head()<import_modules>
test_df = test_df.set_index('Id')
 test_df['Target'] = 0
test_df = test_df[cat_vars+contin_vars+['Target']]
test_X, _, nas, mapper = proc_df(test_df, 'Target', do_scale=True,
mapper=mapper, na_dict=nas )
import numpy as np
import pandas as pd
import os
from tensorflow.python.keras.models import Sequential
from tensorflow.python.keras.layers import Dense, Flatten, GlobalAveragePooling2D, Activation, Flatten, Dropout, BatchNormalization
from tensorflow.python.keras import optimizers, regularizers
from tensorflow.python.keras.applications import InceptionResNetV2
from keras.applications.inception_resnet_v2 import preprocess_input
from tensorflow.python.keras.models import Model
from tensorflow.python.keras.optimizers import sgd
from tensorflow.python.keras.preprocessing.image import ImageDataGenerator<load_from_csv>
opt_params = {'bagging_fraction': 0.8116167224336399,
'colsample_bytree': 0.8368209952651108,
'feature_fraction': 0.5789267873576293,
'lambda_l1': 8.324426408004218,
'lambda_l2': 1.8340450985343382,
'max_depth': 12,
'min_child_samples': 71,
'min_child_weight': 15,
'min_split_gain': 0.08695705870978104,
'num_leaves': 49,
'subsample': 0.8822634555704315}
<choose_model_class><EOS>