Kaggle competition: Costa Rican Household Poverty Level Prediction
all_feat = [col for col in submission.columns if col not in ["sig_id"]]
test_df_copy[all_feat] = test_preds_all.mean(axis=0)
 column_names = []
column_names.append("sig_id")
 column_names.extend(all_feat)
<merge>
df = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/train.csv')
 test = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/test.csv')
 df.shape, test.shape
submission_df = pd.merge(submission['sig_id'] ,test_df_copy[column_names], on="sig_id",how="left")
 submission_df.fillna(0,inplace=True)
 submission_df.head()<set_options>
df_all = df.append(test)
 df_all.shape
os.chdir(".. " )<save_to_csv>
df_all['dependency'].value_counts()
submission_df.to_csv('.. /working/submission.csv', index=None )<create_dataframe>
df_all['edjefa'].value_counts()
<choose_model_class>
df_all['edjefe'].value_counts()
<choose_model_class>
mapeamento = {'yes': 1, 'no': 0}
df_all['edjefa'] = df_all['edjefa'].replace(mapeamento ).astype(int)
 df_all['edjefe'] = df_all['edjefe'].replace(mapeamento ).astype(int )
<create_dataframe>
df_all['dependency'].value_counts()
<concatenate>
df_all['dependency'] = df_all['dependency'].replace(mapeamento ).astype(float )
<create_dataframe>
df_all.isnull().sum()
<concatenate>
df_all[df_all['parentesco1'] == 1]['v2a1'].isnull().sum()
<concatenate>
df_all['v18q'].value_counts()
<create_dataframe>
df_all.isnull().sum().sort_values()
<create_dataframe>
df_all['SQBmeaned'].fillna(-1, inplace=True)
 df_all['meaneduc'].fillna(-1, inplace=True)
 df_all['rez_esc'].fillna(-1, inplace=True )
<define_variables>
feats = [c for c in df_all.columns if c not in ['Id', 'idhogar', 'Target']]
<choose_model_class>
rf = RandomForestClassifier(n_jobs=-1, n_estimators=200, random_state=42 )
<import_modules>
rf.fit(train[feats], train['Target'] )
<compute_test_metric>
test['Target'] = rf.predict(test[feats] ).astype(int )
<choose_model_class>
test['Target'].value_counts(normalize=True )
<prepare_x_and_y>
test[['Id', 'Target']].to_csv('submission.csv', index=False )
<create_dataframe>
train['Target'].value_counts(normalize=True )
<feature_engineering>
heads = train[train['parentesco1'] == 1]
<install_modules>
rf2 = RandomForestClassifier(n_jobs=-1, n_estimators=200, random_state=42)
 rf2.fit(heads[feats], heads['Target'])
 test['Target'] = rf2.predict(test[feats] ).astype(int)
 test[['Id', 'Target']].to_csv('submission.csv', index=False )
!pip install.. /input/pytorch-tabnet/pytorch_tabnet-2.0.1-py3-none-any.whl
!pip install.. /input/iter-strart/iterative_stratification-0.1.6-py3-none-any.whl<set_options>
train['hhsize'].value_counts()
warnings.filterwarnings('ignore' )<set_options>
train['tamviv'].value_counts()
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(33 )<import_modules>
train['tamhog'].value_counts()
class LogitsLogLoss(Metric):
def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):
logits = 1 /(1 + np.exp(-y_pred))
aux =(1-y_true)*np.log(1-logits+1e-15)+ y_true*np.log(logits+1e-15)
 return np.mean(-aux )<load_from_csv>
df_all['hhsize-pc'] = df_all['hhsize'] / df_all['tamviv']
df_all['mobile-pc'] = df_all['qmobilephone'] / df_all['tamviv']
df_all['tablet-pc'] = df_all['v18q1'] / df_all['tamviv']
df_all['rooms-pc'] = df_all['rooms'] / df_all['tamviv']
def load_data(path):
train_feature = pd.read_csv(f'{path}train_features.csv')
 train_target = pd.read_csv(f'{path}train_targets_scored.csv')
 test_feature = pd.read_csv(f'{path}test_features.csv')
 sample_output = pd.read_csv(f'{path}sample_submission.csv')
 features = train_feature.columns[1:]
drug = pd.read_csv(f'{path}train_drug.csv')
 targets = train_target.columns[1:]
train_target = train_target.merge(drug, on='sig_id', how='left')
 return train_feature,train_target,test_feature,sample_output,targets,features<categorify>
feats = [c for c in df_all.columns if c not in ['Id', 'idhogar', 'Target']]
def LabelEncoding(train_feature,test_feature):
le = LabelEncoder()
le1 = LabelEncoder()
le2 = LabelEncoder()
train_feature['cp_type'] = pd.DataFrame(data = le.fit_transform(train_feature['cp_type']))
train_feature['cp_time'] = pd.DataFrame(data = le1.fit_transform(train_feature['cp_time']))
train_feature['cp_dose'] = pd.DataFrame(data = le2.fit_transform(train_feature['cp_dose']))
test_feature['cp_type'] = pd.DataFrame(data = le.transform(test_feature['cp_type']))
test_feature['cp_time'] = pd.DataFrame(data = le1.transform(test_feature['cp_time']))
test_feature['cp_dose'] = pd.DataFrame(data = le2.transform(test_feature['cp_dose']))
return train_feature,test_feature<feature_engineering>
rf3 = RandomForestClassifier(n_jobs=-1, n_estimators=200, random_state=42)
 rf3.fit(train[feats], train['Target'])
 test['Target'] = rf3.predict(test[feats] ).astype(int)
 test[['Id', 'Target']].to_csv('submission.csv', index=False )
def PreProcess(train_feature,test_feature):
GENES = [col for col in train_feature.columns if col.startswith('g-')]
CELLS = [col for col in train_feature.columns if col.startswith('c-')]
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_feature[col].values)
 vec_len_test = len(test_feature[col].values)
 raw_vec = train_feature[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_feature[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_feature[col] = transformer.transform(test_feature[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
n_comp = 600
pca_g = PCA(n_components=n_comp, random_state=42)
 data = pd.concat([pd.DataFrame(train_feature[GENES]), pd.DataFrame(test_feature[GENES])])
 gpca=(pca_g.fit(data[GENES]))
train2=(gpca.transform(train_feature[GENES]))
test2 =(gpca.transform(test_feature[GENES]))
train_gpca = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test_gpca = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_feature = pd.concat(( train_feature, train_gpca), axis=1)
 test_feature = pd.concat(( test_feature, test_gpca), axis=1)
 dump(gpca, open('gpca.pkl', 'wb'))
n_comp = 50
pca_c = PCA(n_components=n_comp, random_state=42)
 data = pd.concat([pd.DataFrame(train_feature[CELLS]), pd.DataFrame(test_feature[CELLS])])
 cpca=(pca_c.fit(data[CELLS]))
train2=(cpca.transform(train_feature[CELLS]))
test2 =(cpca.transform(test_feature[CELLS]))
train_cpca = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test_cpca = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_feature = pd.concat(( train_feature, train_cpca), axis=1)
 test_feature = pd.concat(( test_feature, test_cpca), axis=1)
 dump(cpca, open('cpca.pkl', 'wb'))
c_n = [f for f in list(train_feature.columns)if f not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose']]
mask =(train_feature[c_n].var() >= 0.85 ).values
tmp = train_feature[c_n].loc[:, mask]
train_feature = pd.concat([train_feature[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1)
 tmp = test_feature[c_n].loc[:, mask]
test_feature = pd.concat([test_feature[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1)
 return train_feature,test_feature<concatenate>
heads2 = train[train['parentesco1'] == 1]
def VarThreshold(train_feature,test_feature):
c_n = [f for f in list(train_feature.columns)if f not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose']]
mask =(train_feature[c_n].var() >= 0.85 ).values
tmp = train_feature[c_n].loc[:, mask]
train_feature = pd.concat([train_feature[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1)
 tmp = test_feature[c_n].loc[:, mask]
test_feature = pd.concat([test_feature[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1)
 return train_feature,test_feature<statistical_test>
rf4 = RandomForestClassifier(n_jobs=-1, n_estimators=200, random_state=42)
 rf4.fit(heads2[feats], heads2['Target'])
 test['Target'] = rf4.predict(test[feats] ).astype(int)
 test[['Id', 'Target']].to_csv('submission.csv', index=False )
def Stratify_Drugs(scored, threshold, FOLDS, SEED):
vc = scored.drug_id.value_counts()
vc1 = vc.loc[vc<=threshold].index.sort_values()
vc2 = vc.loc[vc>threshold].index.sort_values()
dct1 = {}; dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=FOLDS, shuffle=True,
random_state=SEED)
 tmp = scored.groupby('drug_id')[targets].mean().loc[vc1]
for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[targets])) :
dd = {k:fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=FOLDS, shuffle=True,
random_state=SEED)
 tmp = scored.loc[scored.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[targets])) :
dd = {k:fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 scored['fold'] = scored.drug_id.map(dct1)
 scored.loc[scored.fold.isna() ,'fold'] = scored.loc[scored.fold.isna() ,'sig_id'].map(dct2)
 scored.fold = scored.fold.astype('int8')
 return scored<drop_column>
rf5 = RandomForestClassifier(n_jobs=-1, n_estimators=200, random_state=42, oob_score=True, class_weight='balanced')
 rf5.fit(train[feats], train['Target'])
 test['Target'] = rf5.predict(test[feats] ).astype(int)
 test[['Id', 'Target']].to_csv('submission.csv', index=False)
def Train_Drug(MAX_EPOCH, FOLDS, train_feature,scored,test_feature):
unused_feat = ['sig_id']
features = [ col for col in train_feature.columns if col not in unused_feat]
train_feature = train_feature[features]
test_feature = test_feature[features]
X_test = test_feature.values
tabnet_params = dict(n_d=24, n_a=24, n_steps=1, gamma=1.3,
lambda_sparse=0, optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2, weight_decay=1e-5),
mask_type='entmax',
scheduler_params=dict(mode="min",
patience=5,
min_lr=1e-5,
factor=0.9,),
scheduler_fn=torch.optim.lr_scheduler.ReduceLROnPlateau,
verbose=10, 
)
 test_preds = []
for i in range(FOLDS):
X_train = train_feature.loc[scored.fold != i].values
y_train = scored.loc[scored.fold != i][targets].values
X_val = train_feature.loc[scored.fold == i].values
y_val = scored.loc[scored.fold == i][targets].values
model = TabNetRegressor(**tabnet_params)
 model.fit(X_train=X_train,
y_train=y_train,
eval_set=[(X_val, y_val)],
eval_name = ["val"],
eval_metric = ["logits_ll"],
max_epochs=MAX_EPOCH,
patience=50, batch_size=1024, virtual_batch_size=128,
num_workers=1, drop_last=False,
loss_fn=torch.nn.functional.binary_cross_entropy_with_logits)
 y_test = model.predict(X_test)
 test_preds.append(1 /(1 + np.exp(-y_test)))
 test_preds_total = np.stack(test_preds)
 return test_preds_total 
<drop_column>
df = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/train.csv')
 test = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/test.csv')
 df.shape, test.shape
def Train_Simple(MAX_EPOCH,FOLDS,SEED,train_feature,train_target,test_feature):
unused_feat = ['sig_id']
features = [ col for col in train_feature.columns if col not in unused_feat]
train_feature = train_feature[features]
train_target = train_target.drop(["sig_id",'drug_id','fold'], axis=1)
 test_feature = test_feature[features]
X_test = test_feature.values
tabnet_params = dict(n_d=24, n_a=24, n_steps=1, gamma=1.3,
lambda_sparse=0, optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2, weight_decay=1e-5),
mask_type='entmax',
scheduler_params=dict(mode="min",
patience=5,
min_lr=1e-5,
factor=0.9,),
scheduler_fn=torch.optim.lr_scheduler.ReduceLROnPlateau,
verbose=10, 
)
 test_preds = []
mskf = MultilabelStratifiedKFold(n_splits=FOLDS, random_state=SEED, shuffle=True)
 for fold_nb,(train_idx, val_idx)in enumerate(mskf.split(train_feature, train_target)) :
print("FOLDS : ", fold_nb)
 X_train, y_train = train_feature.values[train_idx, :], train_target.values[train_idx, :]
X_val, y_val = train_feature.values[val_idx, :], train_target.values[val_idx, :]
model = TabNetRegressor()
model.fit(X_train=X_train,
y_train=y_train,
eval_set=[(X_val, y_val)],
eval_name = ["val"],
eval_metric = ["logits_ll"],
max_epochs=MAX_EPOCH,
patience=20, batch_size=1024, virtual_batch_size=128,
num_workers=1, drop_last=False,
loss_fn=torch.nn.functional.binary_cross_entropy_with_logits)
 y_test = model.predict(X_test)
 test_preds.append(1 /(1 + np.exp(-y_test)))
 test_preds_total = np.stack(test_preds)
 return test_preds_total<categorify>
df_all = df.append(test)
 df_all.shape
threshold = 19
FOLDS = 5
SEED = 33
epochs = 500
train_feature,train_target,test_feature,sample_output,targets,features = load_data('/kaggle/input/lish-moa/')
 train_feature,test_feature = LabelEncoding(train_feature,test_feature)
 train_feature,test_feature = PreProcess(train_feature,test_feature)
 train_feature,test_feature = VarThreshold(train_feature,test_feature)
 scored = Stratify_Drugs(train_target, threshold, FOLDS, SEED)
 pred = Train_Drug(epochs, FOLDS, train_feature,scored,test_feature)
<save_to_csv>
df_all['edjefa'] = df_all['edjefa'].replace(mapeamento ).astype(int)
 df_all['edjefe'] = df_all['edjefe'].replace(mapeamento ).astype(int)
 df_all['dependency'] = df_all['dependency'].replace(mapeamento ).astype(float )
sample_output1 = sample_output.copy()
all_feat = [col for col in sample_output.columns if col not in ["sig_id"]]
sample_output1[all_feat] = pred.mean(axis=0)
 sample_output1.loc[test_feature['cp_type']==0, sample_output.columns[1:]] = 0
sample_output1.to_csv('submission.csv', index=None )<load_from_csv>
df_all['v2a1'].fillna(-1, inplace=True)
 df_all['v18q1'].fillna(0, inplace=True)
 df_all['SQBmeaned'].fillna(-1, inplace=True)
 df_all['meaneduc'].fillna(-1, inplace=True)
 df_all['rez_esc'].fillna(-1, inplace=True )
train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 non_ctl_idx = train_features.loc[train_features['cp_type']!='ctl_vehicle'].index.to_list()
train_features = train_features.drop(['sig_id','cp_type','cp_dose','cp_time'],axis=1)
 train_targets_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 train_targets_scored = train_targets_scored.drop('sig_id',axis=1)
 labels_train = train_targets_scored.values
train_features = train_features.iloc[non_ctl_idx]
labels_train = labels_train[non_ctl_idx]
test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_features = test_features.drop(['sig_id','cp_dose','cp_time'],axis=1)
 json_file_path = '.. /input/t-test-pca-rfe-logistic-regression/main_predictors.json'
with open(json_file_path, 'r')as j:
predictors = json.loads(j.read())
 predictors = predictors['start_predictors']<categorify>
feats = [c for c in df_all.columns if c not in ['Id', 'idhogar', 'Target']]
cs = train_features.columns.str.contains('c-')
 gs = train_features.columns.str.contains('g-')
 def preprocessor(train,test):
n_gs = 2
n_cs = 100
pca_cs = PCA(n_components = n_cs)
 pca_gs = PCA(n_components = n_gs)
 train_pca_gs = pca_gs.fit_transform(train[:,gs])
 train_pca_cs = pca_cs.fit_transform(train[:,cs])
 test_pca_gs = pca_gs.transform(test[:,gs])
 test_pca_cs = pca_cs.transform(test[:,cs])
 train_c_mean = train[:,cs].mean(axis=1)
 test_c_mean = test[:,cs].mean(axis=1)
 train_g_mean = train[:,gs].mean(axis=1)
 test_g_mean = test[:,gs].mean(axis=1)
 train = np.concatenate(( train,train_pca_gs,train_pca_cs,train_c_mean[:,np.newaxis]
,train_g_mean[:,np.newaxis]),axis=1)
 test = np.concatenate(( test,test_pca_gs,test_pca_cs,test_c_mean[:,np.newaxis],
test_g_mean[:,np.newaxis]),axis=1)
 scaler = preprocessing.StandardScaler()
train = scaler.fit_transform(train)
 test = scaler.transform(test)
 return train, test 
<compute_test_metric>
rf6 = RandomForestClassifier(n_jobs=-1, n_estimators=200, random_state=42, oob_score=True, class_weight='balanced')
 rf6.fit(train[feats], train['Target'])
 test['Target'] = rf6.predict(test[feats] ).astype(int)
 test[['Id', 'Target']].to_csv('submission.csv', index=False )
<choose_model_class><EOS> 
rf7 = RandomForestClassifier(max_depth=None, random_state=42, n_jobs=4, n_estimators=700,
min_impurity_decrease=1e-3, min_samples_leaf=2,
verbose=0, class_weight='balanced')
 rf7.fit(train[feats], train['Target'])
 test['Target'] = rf7.predict(test[feats] ).astype(int)
 test[['Id', 'Target']].to_csv('submission.csv', index=False )