Kaggle competition: Costa Rican Household Poverty Level Prediction
from sklearn.decomposition import PCA<define_variables>
df = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/train.csv')
 test = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/test.csv')
 df.shape, test.shape
ncompo_genes = 80
ncompo_cells = 10<define_variables>
df_all = df.append(test)
 df_all.shape
GENES = [col for col in data_all.columns if col.startswith("g-")]
CELLS = [col for col in data_all.columns if col.startswith("c-")]<create_dataframe>
df_all['dependency'].value_counts()
pca_genes = PCA(n_components = ncompo_genes,random_state = 42 ).fit_transform(data_all[GENES])
 pca_cells = PCA(n_components = ncompo_cells,random_state = 42 ).fit_transform(data_all[CELLS])
 pca_genes = pd.DataFrame(pca_genes, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells = pd.DataFrame(pca_cells, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 data_all = pd.concat([data_all, pca_genes, pca_cells], axis = 1 )<feature_engineering>
df_all['edjefa'].value_counts()
GENES = [col for col in data_all.columns if col.startswith("g-")]
CELLS = [col for col in data_all.columns if col.startswith("c-")]
for stats in tqdm.tqdm(["sum", "mean", "std", "kurt", "skew"]):
data_all["g_" + stats] = getattr(data_all[GENES], stats )(axis = 1)
 data_all["c_" + stats] = getattr(data_all[CELLS], stats )(axis = 1)
 data_all["gc_" + stats] = getattr(data_all[GENES + CELLS], stats )(axis = 1 )<categorify>
df_all['edjefe'].value_counts()
data_all = pd.get_dummies(data_all, columns = ["cp_time", "cp_dose"] )<drop_column>
mapeamento = {'yes': 1, 'no': 0}
df_all['edjefa'] = df_all['edjefa'].replace(mapeamento ).astype(int)
 df_all['edjefe'] = df_all['edjefe'].replace(mapeamento ).astype(int )
data_all_copy = data_all.copy()
features_to_drop = ["sig_id"]
data_all.drop(features_to_drop, axis = 1, inplace = True)
 train_df = data_all[: train_features.shape[0]]
train_df.reset_index(drop = True, inplace = True)
 test_df = data_all[train_df.shape[0]: ]
test_df.reset_index(drop = True, inplace = True )<split>
df_all['dependency'].value_counts()
train_df_copy = data_all_copy[: train_features.shape[0]]
train_df_copy.reset_index(drop = True, inplace = True)
 test_df_copy = data_all_copy[train_df.shape[0]: ]
test_df_copy.reset_index(drop = True, inplace = True )<define_variables>
df_all['dependency'] = df_all['dependency'].replace(mapeamento ).astype(float )
train_features = train_df.copy()
test_features = test_df.copy()<init_hyperparams>
df_all.isnull().sum()
BS=1024
MAX_EPOCH=101
N_D = 128
N_A = 48
N_INDEP = 1
N_SHARED = 1
N_STEPS = 3
tabnet_params = dict(n_d=N_D, n_a=N_A, n_steps=N_STEPS,
n_independent=N_INDEP, n_shared=N_SHARED,
lambda_sparse=0., optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2),
mask_type="entmax",
scheduler_params=dict(mode="min",
patience=5,
min_lr=1e-5,
factor=0.9,),
scheduler_fn=torch.optim.lr_scheduler.ReduceLROnPlateau,
verbose=10, 
)
 pretrainer = TabNetPretrainer(**tabnet_params)
 pretrainer.fit(X_train=test_features.values[:,1:],
eval_set=[train_features.values[:,1:]],
max_epochs=MAX_EPOCH,
patience=20, batch_size=BS, virtual_batch_size=128,
num_workers=0, drop_last=True,
pretraining_ratio=0.8 )<prepare_x_and_y>
df_all[df_all['parentesco1'] == 1]['v2a1'].isnull().sum()
X_test = test_features.values[:,1:]<import_modules>
df_all['v18q'].value_counts()
import tensorflow as tf<prepare_x_and_y>
df_all.isnull().sum().sort_values()
scores_auc_all= []
test_cv_preds = []
oof_preds = []
oof_targets = []
scores = []
scores_auc = []
NB_FOLD = 5
for fold_nb in range(NB_FOLD):
train_idx = scored[scored.fold!=fold_nb].index
val_idx = scored[scored.fold==fold_nb].index
print("FOLDS : ", fold_nb)
 if fold_nb >= NB_FOLD:
break
X_train, y_train = train_features.values[train_idx, 1:], train_targets_scored.values[train_idx, 1:].astype(float)
 X_val, y_val = train_features.values[val_idx, 1:], train_targets_scored.values[val_idx, 1:].astype(float)
 MAX_EPOCH=200
BS=1024
tabnet_params = dict(n_d=N_D, n_a=N_A, n_steps=N_STEPS,
n_independent=N_INDEP, n_shared=N_SHARED,
gamma=1.0,
lambda_sparse=0., optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2,
weight_decay=1e-5 
),
mask_type="entmax",
scheduler_params=dict(max_lr=0.05,
steps_per_epoch=int(X_train.shape[0] / BS),
epochs=MAX_EPOCH,
is_batch_level=True),
scheduler_fn=torch.optim.lr_scheduler.OneCycleLR,
verbose=10, 
)
 model = TabNetRegressor(**tabnet_params)
 model.fit(X_train=X_train,
y_train=y_train,
eval_set=[(X_val, y_val)],
eval_name = ["val"],
eval_metric = ["logits_ll"],
max_epochs=MAX_EPOCH,
patience=20, batch_size=BS, virtual_batch_size=128,
num_workers=1, drop_last=True,
from_unsupervised=pretrainer,
loss_fn=torch.nn.functional.binary_cross_entropy_with_logits)
 preds_val = model.predict(X_val)
 preds = 1 /(1 + np.exp(-preds_val))
score = np.min(model.history["val_logits_ll"])
 scores.append(score)
 oof_preds.append(preds)
 oof_targets.append(y_val)
 preds_test = model.predict(X_test)
 test_cv_preds.append(1 /(1 + np.exp(-preds_test)))
 oof_preds_all = np.concatenate(oof_preds)
 oof_targets_all = np.concatenate(oof_targets)
 test_preds_all = np.stack(test_cv_preds )<import_modules>
df_all['SQBmeaned'].fillna(-1, inplace=True)
 df_all['meaneduc'].fillna(-1, inplace=True)
 df_all['rez_esc'].fillna(-1, inplace=True )
from sklearn.metrics import roc_auc_score<compute_test_metric>
feats = [c for c in df_all.columns if c not in ['Id', 'idhogar', 'Target']]
aucs = []
for task_id in range(oof_preds_all.shape[1]):
aucs.append(roc_auc_score(y_true=oof_targets_all[:, task_id],
y_score=oof_preds_all[:, task_id]))
print(f"Overall AUC : {np.mean(aucs)}")
 print(f"Average CV : {np.mean(scores)}" )<feature_engineering>
rf1 = RandomForestClassifier(max_depth=None, random_state=42, n_jobs=4, n_estimators=701,
min_impurity_decrease=1e-3, min_samples_leaf=2,
verbose=0, class_weight='balanced')
 rf1.fit(train[feats], train['Target'])
 test['Target'] = rf1.predict(test[feats] ).astype(int)
 test[['Id', 'Target']].to_csv('submission.csv', index=False )