Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<split>
import numpy as np
import pandas as pd
n_seeds = 7
np.random.seed(1)
 seeds = np.random.randint(0,100,size=n_seeds)
 n_folds = 10
y_pred = np.zeros(( n_test,n_labels))
oof = tf.constant(0.0)
 hists = []
for seed in seeds:
fold = 0
kf = KFold(n_splits=n_folds,shuffle=True,random_state=seed)
 for train, test in kf.split(train_features):
X_train, X_test = preprocessor(train_features.iloc[train].values,
train_features.iloc[test].values)
 _,data_test = preprocessor(train_features.iloc[train].values,
test_features.drop('cp_type',axis=1 ).values)
 X_train_2 = train_features.iloc[train][predictors].values
X_test_2 = train_features.iloc[test][predictors].values
data_test_2 = test_features[predictors].values
y_train = labels_train[train]
y_test = labels_train[test]
n_features = X_train.shape[1]
n_features_2 = X_train_2.shape[1]
model = build_model(n_features, n_features_2, n_labels)
 reduce_lr = callbacks.ReduceLROnPlateau(monitor='val_logloss', factor=0.1, patience=2, mode='min', min_lr=1E-5)
 early_stopping = callbacks.EarlyStopping(monitor='val_logloss', min_delta=1E-5, patience=10, mode='min',restore_best_weights=True)
 hist = model.fit([X_train,X_train_2],y_train, batch_size=128, epochs=192,verbose=0,validation_data =([X_test,X_test_2],y_test),
callbacks=[reduce_lr, early_stopping])
 hists.append(hist)
 model.save('TwoHeads_seed_'+str(seed)+'_fold_'+str(fold))
y_val = model.predict([X_test,X_test_2])
 oof += logloss(tf.constant(y_test,dtype=tf.float32),tf.constant(y_val,dtype=tf.float32)) /(n_folds*n_seeds)
 y_pred += model.predict([data_test,data_test_2])/(n_folds*n_seeds)
 fold += 1<save_to_csv>
train = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/train.csv')
 test = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/test.csv')
 print('Tamanho dos datasets Train e Test:')
 print(' - Train - Linhas:', train.shape[0],'Colunas:', train.shape[1],'
- Test - Linhas:', test.shape[0],'Colunas:', test.shape[1] )
sub = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sub.iloc[:,1:] = y_pred
sub.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0
sub.to_csv('sub2H.csv', index=False )<import_modules>
df_all = train.append(test)
 print('Informações do novo dataset:
df_all - Linhas:', df_all.shape[0],'Colunas:', df_all.shape[1] )
import os
import gc
import pickle
import datetime
import numpy as np
import pandas as pd
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import log_loss
from tqdm.notebook import tqdm
from time import time<load_from_csv>
df_all['edjefa'].value_counts()
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 ss_svc = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 ss_lr = ss_svc.copy()
cols = [c for c in ss_svc.columns.values if c != 'sig_id']
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
def preprocess(df):
df.loc[:, 'cp_type'] = df.loc[:, 'cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 df.loc[:, 'cp_dose'] = df.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 del df['sig_id']
return df
def log_loss_metric(y_true, y_pred):
metrics = []
for _target in train_targets.columns:
metrics.append(log_loss(y_true.loc[:, _target], y_pred.loc[:, _target].astype(float), labels = [0,1]))
return np.mean(metrics)
 train = preprocess(train_features)
 test = preprocess(test_features)
 del train_targets['sig_id']
scaler = StandardScaler()
X = scaler.fit_transform(train.values)
 x_tt = scaler.transform(test_features.values)
 N_STARTS = 3
N_SPLITS = 5
res_svc = train_targets.copy()
ss_svc.loc[:, train_targets.columns] = 0
res_svc.loc[:, train_targets.columns] = 0
for tar in tqdm(range(train_targets.shape[1])) :
start_time = time()
targets = train_targets.values[:, tar]
if targets.sum() >= N_SPLITS:
for seed in range(N_STARTS):
skf = StratifiedKFold(n_splits = N_SPLITS, random_state = seed, shuffle = True)
 for n,(tr, te)in enumerate(skf.split(targets, targets)) :
x_tr, x_val = X[tr], X[te]
y_tr, y_val = targets[tr], targets[te]
model = Ridge(alpha = 0.0001)
 model.fit(x_tr, y_tr)
 ss_svc.loc[:, train_targets.columns[tar]] += model.predict(x_tt)/(N_SPLITS * N_STARTS)
 res_svc.loc[te, train_targets.columns[tar]] += model.predict(x_val)/ N_STARTS
score = log_loss(train_targets.loc[:, train_targets.columns[tar]], res_svc.loc[:, train_targets.columns[tar]])
 print(f'[{str(datetime.timedelta(seconds = time() - start_time)) [2:7]}] SVM Target {tar}:', score)
 print(f'SVM OOF Metric: {log_loss_metric(train_targets, res_svc)}')
 res_svc.loc[train['cp_type'] == 1, train_targets.columns] = 0
ss_svc.loc[test['cp_type'] == 1, train_targets.columns] = 0
print(f'SVM OOF Metric with postprocessing: {log_loss_metric(train_targets, res_svc)}' )<install_modules>
df_all['edjefe'].value_counts()
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet<load_from_csv>
df_all['dependency'].value_counts()
train = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 remove_vehicle = True
if remove_vehicle:
train_features = train.loc[train['cp_type']=='trt_cp'].reset_index(drop=True)
 train_targets_scored = train_targets_scored.loc[train['cp_type']=='trt_cp'].reset_index(drop=True)
 train_targets_nonscored = train_targets_nonscored.loc[train['cp_type']=='trt_cp'].reset_index(drop=True)
 else:
train_features = train<set_options>
mapeamento = {'yes': 1, 'no': 0}
os.environ["CUDA_LAUNCH_BLOCKING"] = "1"
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(42)
 MAX_EPOCH=250
tabnet_params = dict(n_d=24, n_a=24, n_steps=1, gamma=1.3,
lambda_sparse=0, optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2, weight_decay=1e-5),
mask_type='entmax',
scheduler_params=dict(mode="min",
patience=5,
min_lr=1e-5,
factor=0.9,),
scheduler_fn=torch.optim.lr_scheduler.ReduceLROnPlateau,
verbose=10, 
)
 class LogitsLogLoss(Metric):

 def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):

 logits = 1 /(1 + np.exp(-y_pred))
aux =(1-y_true)*np.log(1-logits+1e-15)+ y_true*np.log(logits+1e-15)
 return np.mean(-aux)
 data_path = ".. /input/lish-moa/"<load_from_csv>
df_all['edjefa'] = df_all['edjefa'].replace(mapeamento ).astype(int)
 df_all['edjefe'] = df_all['edjefe'].replace(mapeamento ).astype(int )
train = res_svc[cols].values
X_test = ss_svc[cols].values
train_targets_scored = pd.read_csv(data_path+'train_targets_scored.csv')
 train_targets_scored.drop(columns=["sig_id"], inplace=True)
 submission = pd.read_csv(data_path+'sample_submission.csv')
 scores_auc_all= []
test_cv_preds = []
NB_SPLITS = 10
mskf = MultilabelStratifiedKFold(n_splits=NB_SPLITS, random_state=0, shuffle=True)
 oof_preds = []
oof_targets = []
scores = []
scores_auc = []
for fold_nb,(train_idx, val_idx)in enumerate(mskf.split(train, train_targets_scored)) :
print("FOLDS : ", fold_nb)
 X_train, y_train = train[train_idx, :], train_targets_scored.values[train_idx, :]
X_val, y_val = train[val_idx, :], train_targets_scored.values[val_idx, :]
model = TabNetRegressor(**tabnet_params)
 model.fit(X_train=X_train,
y_train=y_train,
eval_set=[(X_val, y_val)],
eval_name = ["val"],
eval_metric = ["logits_ll"],
max_epochs=MAX_EPOCH,
patience=20, batch_size=1024, virtual_batch_size=128,
num_workers=1, drop_last=False,
loss_fn=torch.nn.functional.binary_cross_entropy_with_logits)
 preds_val = model.predict(X_val)
 preds = 1 /(1 + np.exp(-preds_val))
score = np.min(model.history["val_logits_ll"])
 oof_preds.append(preds_val)
 oof_targets.append(y_val)
 scores.append(score)
 preds_test = model.predict(X_test)
 test_cv_preds.append(1 /(1 + np.exp(-preds_test)))
 oof_preds_all = np.concatenate(oof_preds)
 oof_targets_all = np.concatenate(oof_targets)
 test_preds_all = np.stack(test_cv_preds)
 aucs = []
for task_id in range(oof_preds_all.shape[1]):
aucs.append(roc_auc_score(y_true=oof_targets_all[:, task_id],
y_score=oof_preds_all[:, task_id]))
print(f"Overall AUC : {np.mean(aucs)}")
 print(f"Average CV : {np.mean(scores)}" )<save_to_csv>
df_all['dependency'].value_counts()
all_feat = [col for col in submission.columns if col not in ["sig_id"]]
submission[all_feat] = test_preds_all.mean(axis=0)
 submission.loc[test['cp_type']==1, submission.columns[1:]] = 0
submission.to_csv('subSVM.csv', index=None)
 subSVM = submission.copy()<load_from_csv>
df_all['dependency'] = np.sqrt(df_all['SQBdependency'] )
train = oof
X_test = y_pred
train_targets_scored = pd.read_csv(data_path+'train_targets_scored.csv')
 train_targets_scored.drop(columns=["sig_id"], inplace=True)
 submission = pd.read_csv(data_path+'sample_submission.csv')
 scores_auc_all= []
test_cv_preds = []
NB_SPLITS = 10
mskf = MultilabelStratifiedKFold(n_splits=NB_SPLITS, random_state=0, shuffle=True)
 oof_preds = []
oof_targets = []
scores = []
scores_auc = []
for fold_nb,(train_idx, val_idx)in enumerate(mskf.split(train, train_targets_scored)) :
print("FOLDS : ", fold_nb)
 X_train, y_train = train[train_idx, :], train_targets_scored.values[train_idx, :]
X_val, y_val = train[val_idx, :], train_targets_scored.values[val_idx, :]
model = TabNetRegressor(**tabnet_params)
 model.fit(X_train=X_train,
y_train=y_train,
eval_set=[(X_val, y_val)],
eval_name = ["val"],
eval_metric = ["logits_ll"],
max_epochs=MAX_EPOCH,
patience=20, batch_size=1024, virtual_batch_size=128,
num_workers=1, drop_last=False,
loss_fn=torch.nn.functional.binary_cross_entropy_with_logits)
 preds_val = model.predict(X_val)
 preds = 1 /(1 + np.exp(-preds_val))
score = np.min(model.history["val_logits_ll"])
 oof_preds.append(preds_val)
 oof_targets.append(y_val)
 scores.append(score)
 preds_test = model.predict(X_test)
 test_cv_preds.append(1 /(1 + np.exp(-preds_test)))
 oof_preds_all = np.concatenate(oof_preds)
 oof_targets_all = np.concatenate(oof_targets)
 test_preds_all = np.stack(test_cv_preds)
 aucs = []
for task_id in range(oof_preds_all.shape[1]):
aucs.append(roc_auc_score(y_true=oof_targets_all[:, task_id],
y_score=oof_preds_all[:, task_id]))
print(f"Overall AUC : {np.mean(aucs)}")
 print(f"Average CV : {np.mean(scores)}" )<save_to_csv>
df_all.isnull().sum().sort_values()
all_feat = [col for col in submission.columns if col not in ["sig_id"]]
submission[all_feat] = test_preds_all.mean(axis=0)
 submission.loc[test['cp_type']==1, submission.columns[1:]] = 0
submission.to_csv('subTab2.csv', index=None)
 subTab = submission.copy()<prepare_x_and_y>
df_all['v2a1'] = df_all['v2a1'].fillna(value=df_all['tipovivi3'])
 df_all['v18q1'] = df_all['v18q1'].fillna(value=df_all['v18q'])
 df_all['rez_esc'].fillna(0, inplace=True )
submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 predictions =[]
predictions.append(subSVM)
 predictions.append(sub)
 predictions.append(subTab)
 columns = list(submission.columns)
 columns.remove('sig_id')
 y_pred = pd.DataFrame()
y_pred['sig_id'] = predictions[0]['sig_id']
for column in columns:
column_data = []
for i in range(len(predictions)) :
column_data.append(predictions[i][column])
 y_pred[column] = np.mean(column_data, axis=0)
 y_pred.shape
test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 public_ids = list(submission['sig_id'].values)
 test_ids = list(test['sig_id'].values)
 private_ids = list(set(test_ids)-set(public_ids))
columns = list(submission.columns)
 columns.remove('sig_id')
 submission = pd.DataFrame(index = public_ids + private_ids, columns=columns)
 submission.index.name = 'sig_id'
submission[:] = 0
p_min = 0.0005
p_max = 0.9995
submission.loc[y_pred.sig_id,:] = np.clip(y_pred[columns].values,p_min,p_max)
 submission.loc[test[test.cp_type == 'ctl_vehicle'].sig_id] = 0
submission.to_csv('submission.csv', index=True)
 submission.head().T<set_options>
df_all.isnull().sum().sort_values()
warnings.filterwarnings('ignore' )<install_modules>
df_all['Target'].fillna(-1, inplace=True )
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/ 
<load_from_csv>
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.metrics import confusion_matrix, f1_score, make_scorer
train_features = pd.read_csv(data_dir + 'train_features.csv')
 train_targets_scored = pd.read_csv(data_dir + 'train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv(data_dir + 'train_targets_nonscored.csv')
 train_drug = pd.read_csv(data_dir + 'train_drug.csv')
 test_features = pd.read_csv(data_dir + 'test_features.csv')
 sample_submission = pd.read_csv(data_dir + 'sample_submission.csv')
 print('train_features: {}'.format(train_features.shape))
print('train_targets_scored: {}'.format(train_targets_scored.shape))
print('train_targets_nonscored: {}'.format(train_targets_nonscored.shape))
print('train_drug: {}'.format(train_drug.shape))
print('test_features: {}'.format(test_features.shape))
print('sample_submission: {}'.format(sample_submission.shape))<create_dataframe>
feats = [c for c in df_all.columns if c not in ['Id', 'idhogar', 'Target']]
train_features2=train_features.copy()
test_features2=test_features.copy()<define_variables>
train, test = df_all[df_all['Target'] != -1], df_all[df_all['Target'] == -1]
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<create_dataframe>
rf = RandomForestClassifier(n_jobs=-1, n_estimators=200, oob_score=True, random_state=42 )
qt = QuantileTransformer(n_quantiles=100,random_state=42,output_distribution='normal')
 data = pd.concat([pd.DataFrame(train_features[GENES+CELLS]), pd.DataFrame(test_features[GENES+CELLS])])
 data2 = qt.fit_transform(data[GENES+CELLS])
 train_features[GENES+CELLS] = pd.DataFrame(data2[:train_features.shape[0]])
 test_features[GENES+CELLS] = pd.DataFrame(data2[-test_features.shape[0]:] )<set_options>
rf.fit(train[feats], train['Target'] )
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(42 )<create_dataframe>
test['Target'] = rf.predict(test[feats] ).astype(int )
n_comp = 600
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]];
test2 = data2[-test_features.shape[0]:]
train_gpca = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test_gpca = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_gpca), axis=1)
 test_features = pd.concat(( test_features, test_gpca), axis=1 )<create_dataframe>
test['Target'].value_counts(normalize=True )
n_comp = 50
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train_cpca = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test_cpca = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_cpca), axis=1)
 test_features = pd.concat(( test_features, test_cpca), axis=1 )<create_dataframe>
var_thresh = VarianceThreshold(0.85)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
<concatenate>
df_all['hsize-pc'] = df_all['hhsize'] / df_all['tamviv']
df_all['phone-pc'] = df_all['qmobilephone'] / df_all['tamviv']
df_all['tablets-pc'] = df_all['v18q1'] / df_all['tamviv']
df_all['rooms-pc'] = df_all['rooms'] / df_all['tamviv']
df_all['rent-pc'] = df_all['v2a1'] / df_all['tamviv']
def fe_cluster(train, test, n_clusters_g = 22, n_clusters_c = 4, SEED = 42):
features_g = GENES
features_c = CELLS
def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_{kind}'] = kmeans.labels_[:train.shape[0]]
test[f'clusters_{kind}'] = kmeans.labels_[train.shape[0]:]
train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features2 ,test_features2=fe_cluster(train_features2,test_features2 )<concatenate>
train, test = df_all[df_all['Target'] != -1], df_all[df_all['Target'] == -1]
train_pca=pd.concat(( train_gpca,train_cpca),axis=1)
 test_pca=pd.concat(( test_gpca,test_cpca),axis=1 )<categorify>
retira_cols = ['agesq', 'area2', 'hogar_total', 'male', 'public', 'r4t3', 'tamhog', 'tamviv',
'hhsize', 'v18q', 'v14a', 'mobilephone', 'female']
parentesco_cols = [colunas for colunas in train.columns.tolist() if 'parentesco' in colunas and 'parentesco1' not in colunas]
retira_cols.extend(parentesco_cols)
 train = train.drop(retira_cols, axis=1)
 test = test.drop(retira_cols, axis=1)
 df_all = df_all.drop(retira_cols, axis=1 )
def fe_cluster_pca(train, test,n_clusters=5,SEED = 42):
data=pd.concat([train,test],axis=0)
 kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_pca'] = kmeans.labels_[:train.shape[0]]
test[f'clusters_pca'] = kmeans.labels_[train.shape[0]:]
train = pd.get_dummies(train, columns = [f'clusters_pca'])
 test = pd.get_dummies(test, columns = [f'clusters_pca'])
 return train, test
train_cluster_pca ,test_cluster_pca = fe_cluster_pca(train_pca,test_pca )<split>
feats = [c for c in df_all.columns if c not in ['Id', 'idhogar', 'Target']]
train_cluster_pca = train_cluster_pca.iloc[:,650:]
test_cluster_pca = test_cluster_pca.iloc[:,650:]<split>
heads = train[train['parentesco1'] == 1]
train_features_cluster=train_features2.iloc[:,876:]
test_features_cluster=test_features2.iloc[:,876:]<define_variables>
rf2 = RandomForestClassifier(n_jobs=-1, n_estimators=200, oob_score=True, random_state=42 )
gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577','g-153','g-389','g-60','g-370','g-248','g-167','g-203','g-177','g-301','g-332','g-517','g-6','g-744','g-224','g-162','g-3','g-736','g-486','g-283','g-22','g-359','g-361','g-440','g-335','g-106','g-307','g-745','g-146','g-416','g-298','g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']<feature_engineering>
rf2.fit(train[feats], train['Target'] )
def fe_stats(train, test):
features_g = GENES
features_c = CELLS
for df in train, test:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-23'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
for feature in features_c:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in gsquarecols:
df[f'{feature}_squared'] = df[feature] ** 2
return train, test
train_features2,test_features2=fe_stats(train_features2,test_features2 )<split>
test['Target'] = rf2.predict(test[feats] ).astype(int )
train_features_stats=train_features2.iloc[:,902:]
test_features_stats=test_features2.iloc[:,902:] 
<concatenate>
test['Target'].value_counts(normalize=True )
train_features = pd.concat(( train_features, train_features_cluster,train_cluster_pca,train_features_stats), axis=1)
 test_features = pd.concat(( test_features, test_features_cluster,test_cluster_pca,test_features_stats), axis=1 )<merge>
test[['Id', 'Target']].to_csv('submission.csv', index=False )
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train.merge(train_targets_nonscored, on='sig_id')
 train = train.merge(train_drug, on='sig_id')
 train = train[train['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True )<drop_column>
heads2 = train[train['parentesco1'] == 1]
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )<define_variables>
feats = [c for c in df_all.columns if c not in ['Id', 'idhogar', 'Target']]
target_cols = [x for x in train_targets_scored.columns if x != 'sig_id']
aux_target_cols = [x for x in train_targets_nonscored.columns if x != 'sig_id']
all_target_cols = target_cols + aux_target_cols
num_targets = len(target_cols)
 num_aux_targets = len(aux_target_cols)
 num_all_targets = len(all_target_cols)
 print('num_targets: {}'.format(num_targets))
print('num_aux_targets: {}'.format(num_aux_targets))
print('num_all_targets: {}'.format(num_all_targets))<categorify>
rf3 = RandomForestClassifier(n_jobs=-1, n_estimators=200, oob_score=True, random_state=42 )
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct<train_model>
rf3.fit(heads2[feats], heads2['Target'] )
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds<normalization>
test['Target'] = rf3.predict(test[feats] ).astype(int )
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss<choose_model_class>
test[['Id', 'Target']].to_csv('submission.csv', index=False )
class Model(nn.Module):
def __init__(self, num_features, num_targets):
super(Model, self ).__init__()
self.hidden_size = [1500, 1250, 1000, 750]
self.dropout_value = [0.5, 0.35, 0.3, 0.25]
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.Linear(num_features, self.hidden_size[0])
 self.batch_norm2 = nn.BatchNorm1d(self.hidden_size[0])
 self.dropout2 = nn.Dropout(self.dropout_value[0])
 self.dense2 = nn.Linear(self.hidden_size[0], self.hidden_size[1])
 self.batch_norm3 = nn.BatchNorm1d(self.hidden_size[1])
 self.dropout3 = nn.Dropout(self.dropout_value[1])
 self.dense3 = nn.Linear(self.hidden_size[1], self.hidden_size[2])
 self.batch_norm4 = nn.BatchNorm1d(self.hidden_size[2])
 self.dropout4 = nn.Dropout(self.dropout_value[2])
 self.dense4 = nn.Linear(self.hidden_size[2], self.hidden_size[3])
 self.batch_norm5 = nn.BatchNorm1d(self.hidden_size[3])
 self.dropout5 = nn.Dropout(self.dropout_value[3])
 self.dense5 = nn.utils.weight_norm(nn.Linear(self.hidden_size[3], num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = F.leaky_relu(self.dense3(x))
x = self.batch_norm4(x)
 x = self.dropout4(x)
 x = F.leaky_relu(self.dense4(x))
x = self.batch_norm5(x)
 x = self.dropout5(x)
 x = self.dense5(x)
 return x
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1):
super(LabelSmoothingLoss, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.cls = classes
self.dim = dim
def forward(self, pred, target):
pred = pred.log_softmax(dim=self.dim)
 with torch.no_grad() :
true_dist = torch.zeros_like(pred)
 true_dist.fill_(self.smoothing /(self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
 return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))<choose_model_class>
rf4 = RandomForestClassifier(max_depth=None, random_state=42, n_jobs=4, n_estimators=700,
min_impurity_decrease=1e-3, min_samples_leaf=2,
verbose=0, class_weight='balanced' )
class FineTuneScheduler:
def __init__(self, epochs):
self.epochs = epochs
self.epochs_per_step = 0
self.frozen_layers = []
def copy_without_top(self, model, num_features, num_targets, num_targets_new):
self.frozen_layers = []
model_new = Model(num_features, num_targets)
 model_new.load_state_dict(model.state_dict())
 for name, param in model_new.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == 5:
continue
param.requires_grad = False
if layer_index not in self.frozen_layers:
self.frozen_layers.append(layer_index)
 self.epochs_per_step = self.epochs // len(self.frozen_layers)
 model_new.batch_norm5 = nn.BatchNorm1d(model_new.hidden_size[3])
 model_new.dropout5 = nn.Dropout(model_new.dropout_value[3])
 model_new.dense5 = nn.utils.weight_norm(nn.Linear(model_new.hidden_size[-1], num_targets_new))
model_new.to(DEVICE)
 return model_new
def step(self, epoch, model):
if len(self.frozen_layers)== 0:
return
if epoch % self.epochs_per_step == 0:
last_frozen_index = self.frozen_layers[-1]
for name, param in model.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == last_frozen_index:
param.requires_grad = True
del self.frozen_layers[-1]<categorify>
rf4.fit(heads2[feats], heads2['Target'] )
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data<define_variables>
test['Target'] = rf4.predict(test[feats] ).astype(int )
feature_cols = [c for c in process_data(train ).columns if c not in all_target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold', 'sig_id', 'drug_id']]
num_features = len(feature_cols)
 num_features<define_variables>
test[['Id', 'Target']].to_csv('submission.csv', index=False )
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 24
BATCH_SIZE = 128
WEIGHT_DECAY = {'ALL_TARGETS': 1e-5, 'SCORED_ONLY': 3e-6}
MAX_LR = {'ALL_TARGETS': 1e-2, 'SCORED_ONLY': 3e-3}
DIV_FACTOR = {'ALL_TARGETS': 1e3, 'SCORED_ONLY': 1e2}
PCT_START = 0.1<choose_model_class>
train_y = train['Target']
X_train, X_test, y_train, y_test = train_test_split(train[feats], train_y, test_size=0.2, random_state=42)
 F1_scorer = make_scorer(f1_score, greater_is_better=True, average='macro')
 gbm = lgb.LGBMClassifier(boosting_type='dart', objective='multiclassova', class_weight='balanced', random_state=0)
 gbm.fit(X_train, y_train )
model = Model(num_features, num_all_targets)
 model<features_selection>
train_y = train.Target
X_train, X_test, y_train, y_test = train_test_split(train[feats], train_y, test_size=0.2, random_state=42)
 F1_scorer = make_scorer(f1_score, greater_is_better=True, average='macro')
 gbm = lgb.LGBMClassifier(boosting_type='dart', objective='multiclassova', class_weight='balanced', random_state=0)
 gbm.fit(X_train, y_train )
def make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH):
vc = train.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in SEEDS:
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.loc[train.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 train[kfold_col] = train.drug_id.map(dct1)
 train.loc[train[kfold_col].isna() , kfold_col] = train.loc[train[kfold_col].isna() , 'sig_id'].map(dct2)
 train[kfold_col] = train[kfold_col].astype('int8')
 return train
SEEDS = [3, 4, 5, 6]
NFOLDS = 7
DRUG_THRESH = 18
train = make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH)
 train.head()<train_model>
y_test_pred = gbm.predict(X_test)
 cm = confusion_matrix(y_test, y_test_pred)
 f1 = f1_score(y_test, y_test_pred, average='macro')
 print("confusion matrix:
", cm)
 print("macro F1 score:
", f1 )