Kaggle competition: Costa Rican Household Poverty Level Prediction
data = [
folds,
list(range(len(folds))),
pipes
]
oof_train, oof_test = tabnet_oof(*data, tabnet_params, train_params )<compute_test_metric>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
print(oof_train.mean() , oof_test.mean() , y.mean())
 oof_result(y, oof_train, oof_test )<split>
mapping = {'yes':1,'no':0}
needToChange = train.select_dtypes('object' ).columns[-1:1:-1]
for df in [train,test]:
for col in needToChange:
df[col] = df[col].replace(mapping ).astype(np.float64)
 train.loc[:,needToChange].describe()
prediction = join_control_group(oof_test, test_control, submission, classnames)
 submit_preds(prediction, submission, test_control, classnames)
 pd.read_csv("submission.csv" ).iloc[:5, :5]<import_modules>
test['Target'] = np.nan
data = train.append(test, ignore_index = True )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import tensorflow as tf
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import log_loss
from sklearn.decomposition import PCA
from sklearn.model_selection import train_test_split, KFold
from sklearn.feature_selection import VarianceThreshold
from keras import Sequential
from keras.backend import clear_session
from keras.layers import Dense, Dropout, BatchNormalization, Input
from keras.optimizers import Adam
from keras.losses import BinaryCrossentropy
from tensorflow_addons.layers import WeightNormalization
from tensorflow.keras import callbacks<load_from_csv>
all_equal = train.groupby('idhogar')['Target'].apply(lambda x:x.nunique() == 1)
 not_equal = all_equal[all_equal != True]
print('There are {} households where the family members do not all have the same target.'.format(len(not_equal)) )
train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 train_targets = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 train_non_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_nonscored.csv')
 train_targets.head()<define_variables>
train[train['idhogar'] == not_equal.index[0]][['idhogar', 'parentesco1', 'Target']]
train_features = train_features.set_index('sig_id')
 train_targets = train_targets.set_index('sig_id')
 train_non_scored = train_non_scored.set_index('sig_id' )<define_variables>
household_leader = train.groupby('idhogar')['parentesco1'].sum()
household_nohead = train.loc[train['idhogar'].isin(household_leader[household_leader==0].index),:]
print('There are {} households without a head.'.format(household_nohead['idhogar'].nunique()))
g_features = [x for x in train_features.columns if x.startswith('g-')]
c_features = [x for x in train_features.columns if x.startswith('c-')]
other_features = [x for x in train_features.columns if x not in g_features+c_features]<categorify>
household_nohead_equal = household_nohead.groupby('idhogar')['Target'].apply(lambda x:x.nunique() == 1)
 print('{} Households with no head have different labels.'.format(sum(household_nohead_equal == False)) )
train_features['cp_type'] = train_features['cp_type'].map({
'trt_cp' : 0,
'ctl_vehicle' : 1})
 train_features['cp_dose'] = train_features['cp_dose'].map({
'D1' : 0,
'D2' : 1} )<prepare_x_and_y>
for household in not_equal.index:
true_label = int(train.loc[(train['idhogar'] == household)&(train['parentesco1'] == 1)]['Target'])
 train.loc[train['idhogar'] == household,'Target'] = true_label
all_equal = train.groupby('idhogar')['Target'].apply(lambda x:x.nunique() == 1)
 not_equal = all_equal[all_equal != True]
print('There are {} households where the family members do not all have the same target.'.format(len(not_equal)) )
X = train_features
y = train_targets
y_non_scored = train_non_scored
X = pd.get_dummies(X, columns = ['cp_time'] )<feature_engineering>
missing = pd.DataFrame(data.isnull().sum() ).rename(columns={0:'total'})
 missing['percent'] = missing['total'] / len(data)
 missing.sort_values(by='percent',ascending=False ).head(10 ).drop('Target' )
X['g_std'] = X[g_features].std(axis = 1)
 X['g_var'] = X[g_features].var(axis = 1)
 X['g_skew'] = X[g_features].skew(axis = 1)
 X['g_kurt'] = X[g_features].kurtosis(axis = 1 )<feature_engineering>
heads.groupby('v18q')['v18q1'].apply(lambda x:x.isnull().sum() )
X['c_std'] = X[c_features].std(axis = 1)
 X['c_var'] = X[c_features].var(axis = 1)
 X['c_skew'] = X[c_features].skew(axis = 1)
 X['c_kurt'] = X[c_features].kurtosis(axis = 1 )<define_variables>
data['v18q1'] = data['v18q1'].fillna(0 )
stat_features = ['g_std', 'g_var', 'g_skew', 'g_kurt', 'c_std', 'c_var', 'c_skew', 'c_kurt']<train_on_grid>
data.loc[data['tipovivi1'] == 1,'v2a1'] = 0
data['v2a1-missing'] = data['v2a1'].isnull()
data['v2a1-missing'].value_counts()
selector = VarianceThreshold(0.85)
 selector_cols = g_features + c_features + stat_features
selector.fit(X[selector_cols] )<drop_column>
data.loc[(( data['age'] > 19)|(data['age'] < 7)) &(data['rez_esc'].isnull()), 'rez_esc'] = 0
data['rez_esc-missing'] = data['rez_esc'].isnull()
drop = [col for col, support in zip(selector_cols, selector.get_support())if not support]
X = X.drop(columns = drop )<normalization>
data.loc[data['rez_esc']>5,'rez_esc'] = 5
scale_cols = g_features_train + c_features_train + stat_features_train
scaler = StandardScaler()
X[scale_cols] = scaler.fit_transform(X[scale_cols] )<normalization>
id_ = ['Id', 'idhogar', 'Target']
scale_cols = g_features_train + c_features_train + stat_features_train
scaler = StandardScaler()
X[scale_cols] = scaler.fit_transform(X[scale_cols])
 X.shape<categorify>
ind_bool = ['v18q', 'dis', 'male', 'female', 'estadocivil1', 'estadocivil2', 'estadocivil3',
'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco1', 'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5',
'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12', 'instlevel1', 'instlevel2', 'instlevel3',
'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8',
'instlevel9', 'mobilephone', 'rez_esc-missing']
ind_ordered = ['rez_esc', 'escolari', 'age']
n_components = 100
pca = PCA(n_components = n_components)
 pca_features = pca.fit_transform(X[g_features_train + c_features_train])
 pca_cols = ['pca_'+str(i)for i in range(n_components)]
X[pca_cols] = pca_features<categorify>
hh_bool = ['hacdor', 'hacapo', 'v14a', 'refrig', 'paredblolad', 'paredzocalo',
'paredpreb','pisocemento', 'pareddes', 'paredmad',
'paredzinc', 'paredfibras', 'paredother', 'pisomoscer', 'pisoother',
'pisonatur', 'pisonotiene', 'pisomadera',
'techozinc', 'techoentrepiso', 'techocane', 'techootro', 'cielorazo',
'abastaguadentro', 'abastaguafuera', 'abastaguano',
'public', 'planpri', 'noelec', 'coopele', 'sanitario1',
'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6',
'energcocinar1', 'energcocinar2', 'energcocinar3', 'energcocinar4',
'elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4',
'elimbasu5', 'elimbasu6', 'epared1', 'epared2', 'epared3',
'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3',
'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5',
'computer', 'television', 'lugar1', 'lugar2', 'lugar3',
'lugar4', 'lugar5', 'lugar6', 'area1', 'area2', 'v2a1-missing']
hh_ordered = [ 'rooms', 'r4h1', 'r4h2', 'r4h3', 'r4m1','r4m2','r4m3', 'r4t1', 'r4t2',
'r4t3', 'v18q1', 'tamhog','tamviv','hhsize','hogar_nin',
'hogar_adul','hogar_mayor','hogar_total', 'bedrooms', 'qmobilephone']
hh_cont = ['v2a1', 'dependency', 'edjefe', 'edjefa', 'meaneduc', 'overcrowding']
auto_input_cols = g_features_train + c_features_train
len(auto_input_cols )<choose_model_class>
sqr_ = ['SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe',
'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned', 'agesq']
auto = tf.keras.Sequential([
tf.keras.layers.Dense(400, activation = 'relu', input_shape =(len(auto_input_cols),)) ,
tf.keras.layers.Dense(100, activation = 'relu'),
tf.keras.layers.Dense(400, activation = 'relu'),
tf.keras.layers.Dense(len(auto_input_cols))
])
 auto.compile(optimizer = tf.keras.optimizers.Adam() ,
loss = tf.keras.losses.MeanSquaredError())
 auto.summary()<train_model>
x = ind_bool + ind_ordered + id_ + hh_bool + hh_ordered + hh_cont + sqr_
print('There are no repeats: ', np.all(np.array(list(Counter(x ).values())) == 1))
print('We covered every variable: ', len(x)== data.shape[1] )
auto.fit(X[auto_input_cols], X[auto_input_cols],
epochs = 80,
batch_size = 128 )<categorify>
np.all(np.array(list(Counter(x ).values())) == 1 )
auto_features = tf.keras.backend.function([auto.input], [auto.layers[1].output] )([X[auto_input_cols].to_numpy() , 1])[0]<load_from_csv>
heads = data.loc[data['parentesco1']== 1,:]
heads = heads[id_ + hh_bool + hh_cont + hh_ordered]
heads.shape
test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test = test.set_index('sig_id')
 test['cp_type'] = test['cp_type'].map({
'trt_cp' : 0,
'ctl_vehicle' : 1})
 test['cp_dose'] = test['cp_dose'].map({
'D1' : 0,
'D2' : 1})
 X_test = pd.get_dummies(test, columns = ['cp_time'])
 X_test['g_std'] = X_test[g_features].std(axis = 1)
 X_test['g_var'] = X_test[g_features].var(axis = 1)
 X_test['g_skew'] = X_test[g_features].skew(axis = 1)
 X_test['g_kurt'] = X_test[g_features].kurtosis(axis = 1)
 X_test['c_std'] = X_test[c_features].std(axis = 1)
 X_test['c_var'] = X_test[c_features].var(axis = 1)
 X_test['c_skew'] = X_test[c_features].skew(axis = 1)
 X_test['c_kurt'] = X_test[c_features].kurtosis(axis = 1)
 X_test = X_test.drop(columns = drop)
 X_test[scale_cols] = scaler.transform(X_test[scale_cols])
 pca_features = pca.transform(X_test[g_features_train + c_features_train])
 X_test[pca_cols] = pca_features
auto_features = tf.keras.backend.function([auto.input], [auto.layers[1].output] )([X_test[auto_input_cols].to_numpy() , 1])[0]
X_test[auto_cols] = auto_features
X_test.shape<choose_model_class>
corr_matrix = heads.corr()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape),k=1 ).astype(np.bool))
to_drop = [col for col in upper.columns if any(abs(upper[col])> 0.95)]
to_drop
def build_model(input_shape, output_shape, l1 = 1e-7, l2 = 1e-7, r = 1):

 reg = tf.keras.regularizers.L1L2(l1 = l1, l2 = l2)
 model = Sequential()
model.add(Input(input_shape))
model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(WeightNormalization(Dense(int(2048*r), activation = 'selu',
kernel_initializer = 'lecun_normal',
kernel_regularizer = reg)))
 model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(WeightNormalization(Dense(int(1024*r), activation = 'selu',
kernel_initializer = 'lecun_normal',
kernel_regularizer = reg)))
 model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(WeightNormalization(Dense(int(512*r), activation = 'selu',
kernel_initializer = 'lecun_normal',
kernel_regularizer = reg)))
 model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(WeightNormalization(Dense(int(395*r), activation = 'selu',
kernel_initializer = 'lecun_normal',
kernel_regularizer = reg)))
 model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(BatchNormalization())
 model.add(WeightNormalization(Dense(output_shape, activation = 'sigmoid',
kernel_regularizer = reg)))
 return model<compute_test_metric>
corr_matrix.loc[corr_matrix['tamhog'].abs() >0.9,corr_matrix['tamhog'].abs() >0.9]
def custom_metric(y_true, y_pred):

 eps = tf.constant(1e-7, dtype = tf.float32)
 y_pred = tf.math.maximum(tf.math.minimum(y_pred, 1-eps),eps)
 log_loss = -y_true * tf.math.log(y_pred)-(1 - y_true)* tf.math.log(1 - y_pred)
 return tf.reduce_mean(log_loss )<choose_model_class>
heads = heads.drop(columns = ['tamhog','hogar_total','r4t3'] )
reduce_lr = callbacks.ReduceLROnPlateau(patience = 5, mode = 'min', monitor = 'val_custom_metric', factor=0.1, epsilon=1e-4 )<import_modules>
corr_matrix.loc[corr_matrix['coopele'].abs() > 0.9,corr_matrix['coopele'].abs() > 0.9]
from skopt import gp_minimize
from skopt.space.space import Real, Integer
from skopt.utils import use_named_args<define_search_space>
elec = []
for i,row in heads.iterrows() :
if row['noelec'] == 1:
elec.append(0)
 elif row['coopele'] == 1:
elec.append(1)
 elif row['public'] == 1:
elec.append(2)
 elif row['planpri'] == 1:
elec.append(3)
 else:
elec.append(np.nan)
 heads['elec'] = elec
heads['elec-missing'] = heads['elec'].isnull()
heads = heads.drop(columns = ['noelec','coopele','public','planpri'] )
dim_l1 = Real(low = 1e-7, high = 3e-7, name = 'l1')
 dim_l2 = Real(low = 9e-9, high = 3e-8, name = 'l2')
 dim_r = Real(low = 0.85, high = 1.15, name = 'r')
 dimensions = [dim_l1, dim_l2, dim_r]<split>
corr_matrix.loc[corr_matrix['area2'].abs() >0.9, corr_matrix['area2'].abs() >0.9]
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size = 0.2, shuffle = True )<train_model>
heads = heads.drop(columns='area2')
 heads.groupby('area1')['Target'].value_counts(normalize=True )
@use_named_args(dimensions = dimensions)
 def obj_fun(l1, l2, r):
clear_session()
model = build_model(X_train.shape[1], y_train.shape[1], l1 = l1, l2 = l2, r = r)
 model.compile(optimizer = Adam(2e-3),
loss = BinaryCrossentropy(label_smoothing = 0.001),
metrics = [custom_metric])
 history = model.fit(X_train, y_train,
batch_size = 128,
epochs = 45,
validation_data =(X_val, y_val),
callbacks = [reduce_lr],
verbose = 0)
 return history.history['val_custom_metric'][-1]<init_hyperparams>
heads['roof'] = np.argmax(np.array(heads[['etecho1', 'etecho2', 'etecho3']]),
axis = 1)
 heads = heads.drop(columns = ['etecho1', 'etecho2', 'etecho3'])
 heads['floor'] = np.argmax(np.array(heads[['eviv1', 'eviv2', 'eviv3']]),
axis = 1)
 heads = heads.drop(columns = ['eviv1', 'eviv2', 'eviv3'] )
l1 = 1.3e-07
l2 = 2.27e-08
r = 0.946078<train_model>
count = pd.DataFrame(heads.groupby('wall+roof+floor')['Target'].value_counts(normalize=True)).rename(columns={'Target':'Normalized Count'} ).reset_index()
count.head()
y_preds = []
val_log_losses = []
verbose = 0
batch_size = 128
runs = 5
splits = 7
epochs = 45
for run in range(runs):
kf = KFold(n_splits = splits, shuffle = True)
 for fold,(train_idx, val_idx)in enumerate(kf.split(X, y)) :
print('Run {} Fold {}'.format(run, fold))
X_train, X_val = X.to_numpy() [train_idx], X.to_numpy() [val_idx]
y_train, y_val = y.to_numpy() [train_idx], y.to_numpy() [val_idx]
y_non_scored_train, y_non_scored_val = y_non_scored.to_numpy() [train_idx], y_non_scored.to_numpy() [val_idx]
clear_session()
model = build_model(X_train.shape[1], y_train.shape[1], l1 = l1, l2 = l2, r = r)
 model.compile(optimizer = Adam(2e-3),
loss = BinaryCrossentropy(label_smoothing = 0.001),
metrics = [custom_metric])
 history = model.fit(X_train, y_train,
batch_size = batch_size,
epochs = epochs,
validation_data =(X_val, y_val),
callbacks = [reduce_lr],
verbose = verbose)
 val_log_losses.append(history.history['val_custom_metric'][-1])
 y_preds.append(model.predict(X_test))<compute_test_metric>
heads['warning'] = 1 *(heads['sanitario1'] +(heads['elec'] == 0)+ heads['pisonotiene'] + heads['abastaguano'] +(heads['cielorazo'] == 0))
np.mean(val_log_losses )<prepare_output>
heads['phones-per-capita'] = heads['qmobilephone'] / heads['tamviv']
heads['tablets-per-capita'] = heads['v18q1'] / heads['tamviv']
heads['rooms-per-capita'] = heads['rooms'] / heads['tamviv']
heads['rent-per-capita'] = heads['v2a1'] / heads['tamviv']
predictions = np.mean(np.array(y_preds), axis = 0 )<prepare_output>
corrs = pcorrs.merge(scorrs, on = 'feature')
 corrs['diff'] = corrs['pcorr'] - corrs['scorr']
corrs.sort_values('diff' ).head()
submission = pd.DataFrame(predictions, columns = train_targets.columns)
 submission['sig_id'] = test.index
submission = submission[['sig_id']+list(train_targets.columns)]
submission.loc[list(test.cp_type == 1), train_targets.columns] = 0
print(submission.shape)
 submission.head()<save_to_csv>
corr_matrix = ind.corr()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape),k=1 ).astype(np.bool))
to_drop = [col for col in upper.columns if any(abs(upper[col])> 0.95)]
to_drop
submission.to_csv('submission.csv', index = False )<set_options>
ind = ind.drop(columns = 'male' )
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/
warnings.filterwarnings("ignore")
 os.environ["CUDA_LAUNCH_BLOCKING"] = '1'
plt.style.use("fivethirtyeight")
 c_ = Fore.CYAN
m_ = Fore.MAGENTA
r_ = Fore.RED
b_ = Fore.BLUE
y_ = Fore.YELLOW
g_ = Fore.GREEN 
<set_options>
ind = ind.drop(columns = [c for c in ind if c.startswith('instlevel')])
 ind.shape
seed = 42
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
set_seed(seed )<load_from_csv>
range_ = lambda x:x.max() - x.min()
range_.__name__ = 'range_'
ind_agg = ind.drop(columns='Target' ).groupby('idhogar' ).agg(['min','max','sum','count','std',range_])
 ind_agg.head()
def preprocessor_TabNet() :
data_path = ".. /input/lish-moa/"
no_ctl = True
scale = "rankgauss"
variance_threshould = 0.7
decompo = "PCA"
ncompo_genes = 80
ncompo_cells = 10
encoding = "dummy"
train = pd.read_csv(data_path + "train_features.csv")
 targets = pd.read_csv(data_path + "train_targets_scored.csv")
 test = pd.read_csv(data_path + "test_features.csv")
 train_drug = pd.read_csv(data_path + "train_drug.csv")
 submission = pd.read_csv(data_path + "sample_submission.csv")
 if no_ctl:
print(b_, "not_ctl")
 train = train[train["cp_type"] != "ctl_vehicle"]
test = test[test["cp_type"] != "ctl_vehicle"]
targets = targets.iloc[train.index]
train.reset_index(drop = True, inplace = True)
 test.reset_index(drop = True, inplace = True)
 targets.reset_index(drop = True, inplace = True)
 cols_numeric = [feat for feat in list(train.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
mask =(train[cols_numeric].var() >= variance_threshould ).values
tmp = train[cols_numeric].loc[:, mask]
train = pd.concat([train[["sig_id", "cp_type", "cp_time", "cp_dose"]], tmp], axis = 1)
 cols_numeric = [feat for feat in list(train.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
test = pd.concat([test[["sig_id", "cp_type", "cp_time", "cp_dose"]], test.loc[:,cols_numeric]], axis = 1)
 GENES = [col for col in train.columns if col.startswith("g-")]
CELLS = [col for col in train.columns if col.startswith("c-")]
if scale == "rankgauss":
print(b_, "Rank Gauss")
 for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train[col].values)
 vec_len_test = len(test[col].values)
 raw_vec = train[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test[col] = transformer.transform(test[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
else:
pass
if decompo == "PCA":
print(b_, "PCA")
 pca_genes = PCA(n_components = ncompo_genes, random_state = seed)
 pca_genes_train = pca_genes.fit_transform(train[GENES])
 pca_cells = PCA(n_components = ncompo_cells, random_state = seed)
 pca_cells_train = pca_cells.fit_transform(train[CELLS])
 pca_genes_train = pd.DataFrame(pca_genes_train, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_train = pd.DataFrame(pca_cells_train, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 train = pd.concat([train, pca_genes_train, pca_cells_train], axis = 1)
 pca_genes_test = pca_genes.transform(test[GENES])
 pca_cells_test = pca_cells.transform(test[CELLS])
 pca_genes_test = pd.DataFrame(pca_genes_test, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_test = pd.DataFrame(pca_cells_test, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 test = pd.concat([test, pca_genes_test, pca_cells_test], axis = 1)
 else:
pass
if encoding == "dummy":
print(b_, "One-Hot")
 train = pd.get_dummies(train, columns = ["cp_time", "cp_dose"])
 test = pd.get_dummies(test, columns = ["cp_time", "cp_dose"])
 else:
pass
for stats in tqdm.tqdm(["sum", "mean", "std", "kurt", "skew"]):
train["g_" + stats] = getattr(train[GENES], stats )(axis = 1)
 train["c_" + stats] = getattr(train[CELLS], stats )(axis = 1)
 train["gc_" + stats] = getattr(train[GENES + CELLS], stats )(axis = 1)
 test["g_" + stats] = getattr(test[GENES], stats )(axis = 1)
 test["c_" + stats] = getattr(test[CELLS], stats )(axis = 1)
 test["gc_" + stats] = getattr(test[GENES + CELLS], stats )(axis = 1)
 gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577',
'g-153','g-389','g-60','g-370','g-248','g-167',
'g-203','g-177','g-301','g-332','g-517','g-6',
'g-744','g-224','g-162','g-3','g-736','g-486',
'g-283','g-22','g-359','g-361','g-440','g-335',
'g-106','g-307','g-745','g-146','g-416','g-298',
'g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']
for df in [train, test]:
df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-23'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
for feature in gsquarecols:
if feature in GENES:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in CELLS:
df[f'{feature}_squared'] = df[feature] ** 2
train = train.merge(targets, on='sig_id')
 train = train.merge(train_drug, on='sig_id')
 target_cols = [x for x in targets.columns if x != 'sig_id']
train_df = train
train_df.reset_index(drop=True, inplace=True)
 test.reset_index(drop = True, inplace = True)
 features_to_drop = ["sig_id", "cp_type"]
test.drop(features_to_drop, axis = 1, inplace = True)
 feature_cols = [c for c in train_df.columns if c not in targets.columns]
feature_cols = [c for c in feature_cols if c not in [ 'sig_id', 'drug_id', 'cp_type']]
return train_df, targets, test, submission, target_cols, feature_cols<prepare_x_and_y>
new_cols = []
for col in ind_agg.columns.levels[0]:
for stat in ind_agg.columns.levels[1]:
new_cols.append(f'{col}-{stat}')
 ind_agg.columns = new_cols
ind_agg.head()
SEEDS = [x for x in range(42,49)]
NFOLDS = 10
X_test = test_TabNet.values<choose_model_class>
ind_agg = ind_agg.drop(columns=to_drop)
 ind_feats = list(ind_agg.columns)
 final = heads.merge(ind_agg,on='idhogar',how='left')
 print('Final features shape: ', final.shape )
MAX_EPOCH = 200
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 p_min = 0.0005
p_max = 0.9995
def InferenceTabNet(X_test, SEEDS, NFOLDS,DEVICE):
test_cv_preds = []
for s in SEEDS:
for fold_id in range(NFOLDS):
print(b_,"FOLDS: ", r_, fold_id + 1, y_, 'seed:', s)
 print(g_, '*' * 60, c_)
 saved_path_name = '.. /input/tabnetfe7seeds424810folds/TabNet_seed_'\
+str(s)+'_fold_'+str(fold_id+1)
 saved_file = '/kaggle/working/model'
shutil.make_archive(saved_file,'zip',saved_path_name)
 print(f'Successfully saved model at {saved_file}.zip')
 with zipfile.ZipFile('model.zip')as z:
with z.open('model_params.json')as f:
loaded_params = json.load(f)
 with z.open('network.pt')as f:
try:
saved_state_dict = torch.load(f, map_location=DEVICE)
 except io.UnsupportedOperation:
saved_state_dict = torch.load(io.bytesIO(f.read()),map_location = DEVICE)
 loaded_model = TabNetRegressor(**loaded_params)
 loaded_model._set_network()
loaded_model.network.load_state_dict(saved_state_dict)
 loaded_model.network.eval()
preds_test = loaded_model.predict(X_test)
 test_cv_preds.append(1 /(1 + np.exp(-preds_test)))
 test_preds_all = np.stack(test_cv_preds)
 saved_path_name = '.. /input/tabnetfe7seeds424810folds/'
oof_TabNet_all = np.load(saved_path_name + 'oof_TabNet_all.npy')
 return oof_TabNet_all,test_preds_all
oof_TabNet_all,test_preds_all = InferenceTabNet(X_test, SEEDS, NFOLDS,DEVICE )<load_from_csv>
head_gender = ind.loc[ind['parentesco1'] == 1, ['idhogar', 'female']]
final = final.merge(head_gender, on = 'idhogar', how = 'left' ).rename(columns = {'female': 'female-head'} )
def to_submission_TabNet(test_preds_all,submission_TabNet):
data_path = ".. /input/lish-moa/"
all_feat = [col for col in submission_TabNet.columns if col not in ["sig_id"]]
test_TabNet = pd.read_csv(data_path + "test_features.csv")
 sig_id = test_TabNet[test_TabNet["cp_type"] != "ctl_vehicle"].sig_id.reset_index(drop = True)
 tmp = pd.DataFrame(test_preds_all.mean(axis = 0), columns = all_feat)
 tmp["sig_id"] = sig_id
submission_TabNet = pd.merge(test_TabNet[["sig_id"]], tmp, on = "sig_id", how = "left")
 submission_TabNet.fillna(0, inplace = True)
 submission_TabNet.to_csv("submission_TabNet.csv", index = None)
 return submission_TabNet
submission_TabNet = to_submission_TabNet(test_preds_all,submission_TabNet)
 display(submission_TabNet.head())
 print(f"{b_}submission_TabNet.shape: {r_}{submission_TabNet.shape}" )<set_options>
final.groupby('female-head')['Target'].value_counts(normalize=True )
%who DataFrame
del targets_TabNet, test_TabNet, train_df,target_cols, feature_cols
gc.collect()
%who DataFrame<set_options>
final.groupby('female-head')['meaneduc'].agg(['mean', 'count'] )
seed = 42
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(seed)
 def preprocessor_nn_transfer() :
variance_threshould = 0.8
ncompo_genes = 600
ncompo_cells = 50
data_dir = '.. /input/lish-moa/'
train_features = pd.read_csv(data_dir + 'train_features.csv')
 train_targets_scored = pd.read_csv(data_dir + 'train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv(data_dir + 'train_targets_nonscored.csv')
 train_drug = pd.read_csv(data_dir + 'train_drug.csv')
 test_features = pd.read_csv(data_dir + 'test_features.csv')
 sample_submission = pd.read_csv(data_dir + 'sample_submission.csv')
 train_features = train_features[train_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 test_features = test_features[test_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 train_features = train_features.drop('cp_type', axis=1)
 test_features = test_features.drop('cp_type', axis=1)
 GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
pca_genes = PCA(n_components = ncompo_genes, random_state = seed)
 pca_genes_train = pca_genes.fit_transform(train_features[GENES])
 pca_cells = PCA(n_components = ncompo_cells, random_state = seed)
 pca_cells_train = pca_cells.fit_transform(train_features[CELLS])
 pca_genes_train = pd.DataFrame(pca_genes_train, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_train = pd.DataFrame(pca_cells_train, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 train_features = pd.concat([train_features, pca_genes_train, pca_cells_train], axis = 1)
 pca_genes_test = pca_genes.transform(test_features[GENES])
 pca_cells_test = pca_cells.transform(test_features[CELLS])
 pca_genes_test = pd.DataFrame(pca_genes_test, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_test = pd.DataFrame(pca_cells_test, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 test_features = pd.concat([test_features, pca_genes_test, pca_cells_test], axis = 1)
 PCA')
 cols_numeric = [feat for feat in list(train_features.columns)if feat not in ["sig_id", "cp_time", "cp_dose"]]
mask =(train_features[cols_numeric].var() >= variance_threshould ).values
tmp = train_features[cols_numeric].loc[:, mask]
train_features = pd.concat([train_features[["sig_id", "cp_time", "cp_dose"]], tmp], axis = 1)
 cols_numeric = [feat for feat in list(train_features.columns)if feat not in ["sig_id", "cp_time", "cp_dose"]]
test_features = pd.concat([test_features[["sig_id", "cp_time", "cp_dose"]], test_features.loc[:,cols_numeric]], axis = 1)
 train_features = pd.get_dummies(train_features, columns = ['cp_time', 'cp_dose'])
 test_features = pd.get_dummies(test_features, columns = ['cp_time', 'cp_dose'])
 Feature selection')
 train = train_features.merge(train_targets_scored, on='sig_id')
 train = train.merge(train_targets_nonscored, on='sig_id')
 train = train.merge(train_drug, on='sig_id')
 test = test_features
Join')
 gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577',
'g-153','g-389','g-60','g-370','g-248','g-167',
'g-203','g-177','g-301','g-332','g-517','g-6',
'g-744','g-224','g-162','g-3','g-736','g-486',
'g-283','g-22','g-359','g-361','g-440','g-335',
'g-106','g-307','g-745','g-146','g-416','g-298',
'g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']
for df in [train, test]:
df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-23'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
for feature in gsquarecols:
if feature in GENES:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in CELLS:
df[f'{feature}_squared'] = df[feature] ** 2
target_cols = [x for x in train_targets_scored.columns if x != 'sig_id']
aux_target_cols = [x for x in train_targets_nonscored.columns if x != 'sig_id']
all_target_cols = target_cols + aux_target_cols
num_targets = len(target_cols)
 num_aux_targets = len(aux_target_cols)
 num_all_targets = len(all_target_cols)
 num_targets: {}'.format(num_targets))
')
 return train, test, num_targets, num_aux_targets, num_all_targets, target_cols, aux_target_cols, all_target_cols<split>
scorer = make_scorer(f1_score, greater_is_better=True, average = 'macro' )
train, test, num_targets, num_aux_targets, num_all_targets, target_cols, aux_target_cols, all_target_cols = preprocessor_nn_transfer()<categorify>
train_labels = np.array(list(final.loc[final['Target'].notnull() ,'Target'].astype(np.uint8)))
 train_set = final[final['Target'].notnull() ].drop(columns=['Id','idhogar','Target'])
 test_set = final[final['Target'].isnull() ].drop(columns=['Id','idhogar','Target'])
 submission_base = test[['Id', 'idhogar']].copy()
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
class Model(nn.Module):
def __init__(self, num_features, num_targets):
super(Model, self ).__init__()
self.hidden_size = [1500, 1250, 1000, 750]
self.dropout_value = [0.5, 0.35, 0.3, 0.25]
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.Linear(num_features, self.hidden_size[0])
 self.batch_norm2 = nn.BatchNorm1d(self.hidden_size[0])
 self.dropout2 = nn.Dropout(self.dropout_value[0])
 self.dense2 = nn.Linear(self.hidden_size[0], self.hidden_size[1])
 self.batch_norm3 = nn.BatchNorm1d(self.hidden_size[1])
 self.dropout3 = nn.Dropout(self.dropout_value[1])
 self.dense3 = nn.Linear(self.hidden_size[1], self.hidden_size[2])
 self.batch_norm4 = nn.BatchNorm1d(self.hidden_size[2])
 self.dropout4 = nn.Dropout(self.dropout_value[2])
 self.dense4 = nn.Linear(self.hidden_size[2], self.hidden_size[3])
 self.batch_norm5 = nn.BatchNorm1d(self.hidden_size[3])
 self.dropout5 = nn.Dropout(self.dropout_value[3])
 self.dense5 = nn.utils.weight_norm(nn.Linear(self.hidden_size[3], num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = F.leaky_relu(self.dense3(x))
x = self.batch_norm4(x)
 x = self.dropout4(x)
 x = F.leaky_relu(self.dense4(x))
x = self.batch_norm5(x)
 x = self.dropout5(x)
 x = self.dense5(x)
 return x
class FineTuneScheduler:
def __init__(self, epochs):
self.epochs = epochs
self.epochs_per_step = 0
self.frozen_layers = []
def copy_without_top(self, model, num_features, num_targets, num_targets_new):
self.frozen_layers = []
model_new = Model(num_features, num_targets)
 model_new.load_state_dict(model.state_dict())
 for name, param in model_new.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == 5:
continue
param.requires_grad = False
if layer_index not in self.frozen_layers:
self.frozen_layers.append(layer_index)
 self.epochs_per_step = self.epochs // len(self.frozen_layers)
 model_new.batch_norm5 = nn.BatchNorm1d(model_new.hidden_size[3])
 model_new.dropout5 = nn.Dropout(model_new.dropout_value[3])
 model_new.dense5 = nn.utils.weight_norm(nn.Linear(model_new.hidden_size[-1], num_targets_new))
model_new.to(DEVICE)
 return model_new
def step(self, epoch, model):
if len(self.frozen_layers)== 0:
return
if epoch % self.epochs_per_step == 0:
last_frozen_index = self.frozen_layers[-1]
for name, param in model.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == last_frozen_index:
param.requires_grad = True
del self.frozen_layers[-1]<define_variables>
features = list(train_set.columns)
 pipeline = Pipeline([('imputer',Imputer(strategy='median')) ,('scaler',MinMaxScaler())])
 train_set = pipeline.fit_transform(train_set)
 test_set = pipeline.transform(test_set )
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 24
BATCH_SIZE = 128
WEIGHT_DECAY = {'ALL_TARGETS': 1e-5, 'SCORED_ONLY': 3e-6}
MAX_LR = {'ALL_TARGETS': 1e-2, 'SCORED_ONLY': 3e-3}
DIV_FACTOR = {'ALL_TARGETS': 1e3, 'SCORED_ONLY': 1e2}
PCT_START = 0.1
SEEDS = list(range(42,49))
NFOLDS = 10
DRUG_THRESH = 18
feature_cols = [c for c in train.columns if c not in all_target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold', 'sig_id', 'drug_id']]
num_features = len(feature_cols)
<choose_model_class>
model = RandomForestClassifier(n_estimators=100, random_state=10, n_jobs = -1)
 cv_score = cross_val_score(model, train_set, train_labels, cv = 10, scoring = scorer)
 print(f'10 Fold Cross Validation F1 Score = {round(cv_score.mean() , 4)} with std = {round(cv_score.std() , 4)}' )
def run_training(fold_id, seed_id):
seed_everything(seed_id)
 test_ = test
model = Model(num_features, num_targets)
 model.load_state_dict(torch.load(f".. /input/nn-transferlearning-7seeds4248-10folds/SCORED_ONLY_SEED_{seed_id}_FOLD{fold_id}_.pth"))
model.to(DEVICE)
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 predictions = np.zeros(( len(test_), num_targets))
predictions = inference_fn(model, testloader, DEVICE)
 print(f"Pretrained_Model_Loaded_SCORED_ONLYSEED: {seed_id}, FOLD: {fold_id} ")
 return predictions
def run_k_fold(NFOLDS, seed_id):
predictions = np.zeros(( len(test), len(target_cols)))
 for fold_id in range(NFOLDS):
pred_ = run_training(fold_id, seed_id)
 predictions += pred_ / NFOLDS
return predictions<feature_engineering>
model.fit(train_set,train_labels)
 indices = np.argsort(model.feature_importances_)[::-1]
importances = model.feature_importances_
for i in range(len(features)) :
print("%d)feature importances of %s %f"%(i,features[indices[i]],importances[indices[i]]))
predictions = np.zeros(( len(test), len(target_cols)))
 time_begin = time()
def Inference_Transfer_learning(predictions,SEEDS,NFOLDS):
for seed_id in SEEDS:
predictions_ = run_k_fold(NFOLDS, seed_id)
 predictions += predictions_ / len(SEEDS)
 return predictions
predictions = Inference_Transfer_learning(predictions,SEEDS,NFOLDS)
 time_diff = time() - time_begin
test[target_cols] = predictions
str(timedelta(seconds=time_diff))<save_to_csv>
temp_df = pd.DataFrame({'feature':features,'importance':importances} )
def to_submission_Transfer_learning(target_cols,test):
data_dir = '.. /input/lish-moa/'
sample_submission = pd.read_csv(data_dir + 'sample_submission.csv')
 sub_nn_transfer = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub_nn_transfer.to_csv('submission_nn_transfer.csv', index=False)
 return sub_nn_transfer
sub_nn_transfer = to_submission_Transfer_learning(target_cols,test)
 print(sub_nn_transfer.shape)
 display(sub_nn_transfer.head() )<set_options>
from sklearn.svm import LinearSVC
from sklearn.naive_bayes import GaussianNB
from sklearn.neural_network import MLPClassifier
from sklearn.linear_model import LogisticRegressionCV, RidgeClassifierCV
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.neighbors import KNeighborsClassifier
%who DataFrame
del test, train
gc.collect()
%who DataFrame<load_from_csv>
model_results = cv_model(train_set,train_labels,LinearSVC() ,'LSVC',model_results )
def readin_data_resnet() :
train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 train_drug = pd.read_csv('.. /input/lish-moa/train_drug.csv')
 non_ctl_idx = train_features.loc[train_features['cp_type']!='ctl_vehicle'].index.to_list()
train_features = train_features.drop(['sig_id','cp_type','cp_dose','cp_time'],axis=1)
 train_targets_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 targets_drug = train_targets_scored.merge(train_drug, on='sig_id')
 train_targets_scored = train_targets_scored.drop('sig_id',axis=1)
 labels_train = train_targets_scored.values
train_features = train_features.iloc[non_ctl_idx]
labels_train = labels_train[non_ctl_idx]
targets_drug = targets_drug.iloc[non_ctl_idx]
test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_features = test_features.drop(['sig_id','cp_dose','cp_time'],axis=1)
 json_file_path = '.. /input/ttestpcarfelogisticregression/main_predictors.json'
with open(json_file_path, 'r')as j:
predictors = json.loads(j.read())
 predictors = predictors['start_predictors']
cs = train_features.columns.str.contains('c-')
 gs = train_features.columns.str.contains('g-')
 return train_features,labels_train,test_features,predictors,cs,gs 
<compute_test_metric>
model_results = cv_model(train_set,train_labels,GaussianNB() ,'GNB',model_results )
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -backend.mean(y_true*backend.log(y_pred)+(1-y_true)*backend.log(1-y_pred))
def resnet_preprocessor(train,test,cs,gs):
n_gs = 2
n_cs = 100
pca_cs = PCA(n_components = n_cs)
 pca_gs = PCA(n_components = n_gs)
 train_pca_gs = pca_gs.fit_transform(train[:,gs])
 train_pca_cs = pca_cs.fit_transform(train[:,cs])
 test_pca_gs = pca_gs.transform(test[:,gs])
 test_pca_cs = pca_cs.transform(test[:,cs])
 train_c_mean = train[:,cs].mean(axis=1)
 test_c_mean = test[:,cs].mean(axis=1)
 train_g_mean = train[:,gs].mean(axis=1)
 test_g_mean = test[:,gs].mean(axis=1)
 train = np.concatenate(( train,train_pca_gs,train_pca_cs,train_c_mean[:,np.newaxis]
,train_g_mean[:,np.newaxis]),axis=1)
 test = np.concatenate(( test,test_pca_gs,test_pca_cs,test_c_mean[:,np.newaxis],
test_g_mean[:,np.newaxis]),axis=1)
 scaler = preprocessing.QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 train = scaler.fit_transform(train)
 test = scaler.transform(test)
 return train, test 
<choose_model_class>
model_results = cv_model(train_set,train_labels,MLPClassifier(hidden_layer_sizes=(32,64,128,64,32)) ,'MLP',model_results )
def resnet_model(n_features, n_features_2, n_labels, label_smoothing = 0.0005):
input_1 = layers.Input(shape =(n_features,), name = 'Input1')
 input_2 = layers.Input(shape =(n_features_2,), name = 'Input2')
 head_1 = Sequential([
layers.BatchNormalization() ,
layers.Dropout(0.25),
layers.Dense(512, activation="elu"),
layers.BatchNormalization() ,
layers.Dense(256, activation = "elu")
 ],name='Head1')
 input_3 = head_1(input_1)
 input_3_concat = layers.Concatenate()([input_2, input_3])
 head_2 = Sequential([
layers.BatchNormalization() ,
layers.Dropout(0.3),
layers.Dense(512, "relu"),
layers.BatchNormalization() ,
layers.Dense(512, "elu"),
layers.BatchNormalization() ,
layers.Dense(256, "relu"),
layers.BatchNormalization() ,
layers.Dense(256, "elu")
 ],name='Head2')
 input_4 = head_2(input_3_concat)
 input_4_avg = layers.Average()([input_3, input_4])
 head_3 = Sequential([
layers.BatchNormalization() ,
layers.Dense(256, kernel_initializer='lecun_normal', activation='selu'),
layers.BatchNormalization() ,
layers.Dense(n_labels, kernel_initializer='lecun_normal', activation='selu'),
layers.BatchNormalization() ,
layers.Dense(n_labels, activation="sigmoid")
 ],name='Head3')
 output = head_3(input_4_avg)
 model = Model(inputs = [input_1, input_2], outputs = output)
 model.compile(optimizer='adam', loss=losses.BinaryCrossentropy(label_smoothing=label_smoothing), metrics=logloss)
 return model<find_best_model_class>
model_results = cv_model(train_set, train_labels,
LinearDiscriminantAnalysis() ,
'LDA', model_results )
def predict_resnet(train_features,test_features,labels_train, SEEDS, NFOLDS,cs,gs):
n_labels = labels_train.shape[1]
n_train = train_features.shape[0]
n_test = test_features.shape[0]
n_seeds = len(SEEDS)
 n_folds = NFOLDS
y_pred = np.zeros(( n_test,n_labels))
_,data_test = resnet_preprocessor(train_features.values,
test_features.drop('cp_type',axis=1 ).values,cs,gs)
 data_test_2 = test_features[predictors].values
for seed in SEEDS:
for fold in range(NFOLDS):
K.clear_session()
model = keras.models.load_model('/kaggle/input/resent-1841/TwoHeads_seed_'+str(seed)+'_fold_'+str(fold), custom_objects={'logloss':logloss})
 y_pred += model.predict([data_test,data_test_2])/(n_folds*n_seeds)
 print(f"Pretrained_Resnet_Model: {seed}, FOLD: {fold} ")
 return y_pred 
<predict_on_test>
model_results = cv_model(train_set, train_labels, RidgeClassifierCV() , 'RIDGE', model_results )
train_features,labels_train,test_features,predictors,cs,gs = readin_data_resnet()
p_min = 0.0005
p_max = 0.9995
DRUG_THRESH = 18
n_seeds = 7
np.random.seed(42)
 SEEDS = np.random.randint(0,100,size=n_seeds)
 NFOLDS = 10
test_pred_rs = predict_resnet(train_features,test_features,labels_train,SEEDS, NFOLDS,cs,gs )<save_to_csv>
for n in [5, 10, 20]:
print(f'
KNN with {n} neighbors
')
 model_results = cv_model(train_set, train_labels,
KNeighborsClassifier(n_neighbors = n),
f'knn-{n}', model_results )
sub_rs = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sub_rs.iloc[:,1:] = np.zeros([test_pred_rs.shape[0],test_pred_rs.shape[1]])
 sub_rs.iloc[:,1:] = test_pred_rs
sub_rs.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0
sub_rs.to_csv('submission_rs.csv', index=False)
<create_dataframe>
model_results = cv_model(train_set, train_labels,
ExtraTreesClassifier(n_estimators = 100, random_state = 10),
'EXT', model_results )
%who DataFrame
del train_features,labels_train,test_features,predictors,cs,gs
gc.collect()
%who DataFrame<normalization>
model_results = cv_model(train_set, train_labels,
RandomForestClassifier(n_estimators = 100, random_state = 10),
'RF', model_results )
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
def remove_ctl(no_ctl,train,test,targets):
if no_ctl:
print('not_ctl')
 train = train[train['cp_type']!='ctl_vehicle']
test = test[test['cp_type']!='ctl_vehicle']
targets = targets.iloc[train.index]
train.reset_index(drop=True, inplace=True)
 test.reset_index(drop=True, inplace=True)
 targets.reset_index(drop=True, inplace=True)
 return train,test,targets
def remove_small_variance(variance_threshould,train,test):
cols_numeric = [feat for feat in list(train.columns)if feat not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose']]
mask =(train[cols_numeric].var() >= variance_threshould ).values
tmp_train = train[cols_numeric].loc[:, mask]
tmp_test = test[cols_numeric].loc[:, mask]
train = pd.concat([train[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp_train], axis=1)
 test = pd.concat([test[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp_test], axis=1)
 return train,test
def pca_process(ncompo_genes,ncompo_cells,train,test):
GENES = [col for col in train.columns if col.startswith('g-')]
CELLS = [col for col in train.columns if col.startswith('c-')]
pca_genes = PCA(n_components=ncompo_genes, random_state=base_seed ).fit(train[GENES])
 pca_cells = PCA(n_components=ncompo_cells, random_state=base_seed ).fit(train[CELLS])
 train_pca_gene = pca_genes.transform(train[GENES])
 test_pca_gene = pca_genes.transform(test[GENES])
 train_pca_cell = pca_cells.transform(train[CELLS])
 test_pca_cell = pca_cells.transform(test[CELLS])
 train_pca_gene = pd.DataFrame(train_pca_gene, columns=[f'pca_g-{i}' for i in range(ncompo_genes)])
 test_pca_gene = pd.DataFrame(test_pca_gene, columns=[f'pca_g-{i}' for i in range(ncompo_genes)])
 train_pca_cell = pd.DataFrame(train_pca_cell, columns=[f'pca_c-{i}' for i in range(ncompo_cells)])
 test_pca_cell = pd.DataFrame(test_pca_cell, columns=[f'pca_c-{i}' for i in range(ncompo_cells)])
 train_df = pd.concat([train, train_pca_gene, train_pca_cell], axis=1)
 test_df = pd.concat([test, test_pca_gene, test_pca_cell], axis=1)
 return train_df,test_df
def scale_data(scale, train_df,test_df):
if scale == 'minmax':
print('minmax')
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_minmax, axis=0)
 elif scale == 'rankgauss':
print('rankgauss')
 cols_numeric = [feat for feat in list(train_df.columns)if feat not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose'] and "pca_" not in feat]
scaler = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 train_df[cols_numeric] = scaler.fit_transform(train_df[cols_numeric])
 test_df[cols_numeric] = scaler.transform(test_df[cols_numeric])
 else:
pass
return train_df,test_df
def add_features(train,test):
GENES = [col for col in train.columns if col.startswith('g-')]
CELLS = [col for col in train.columns if col.startswith('c-')]
for stats in tqdm.tqdm(['sum', 'mean', 'std', 'kurt', 'skew']):
train['g_'+stats] = getattr(train[GENES], stats )(axis=1)
 train['c_'+stats] = getattr(train[CELLS], stats )(axis=1)
 train['gc_'+stats] = getattr(train[GENES+CELLS], stats )(axis=1)
 test['g_'+stats] = getattr(test[GENES], stats )(axis=1)
 test['c_'+stats] = getattr(test[CELLS], stats )(axis=1)
 test['gc_'+stats] = getattr(test[GENES+CELLS], stats )(axis=1)
 return train,test
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
def data_processing(train,test, targets):
train,test,targets = remove_ctl(no_ctl,train,test,targets)
 print("remove control: ", train.shape,test.shape)
 train,test = remove_small_variance(variance_threshould,train,test)
 print("remove small_var: ", train.shape,test.shape)
 train_df,test_df = pca_process(ncompo_genes,ncompo_cells,train,test)
 print("do PCA: ", train_df.shape,test_df.shape)
 train_df,test_df = scale_data(scale, train_df,test_df)
 print("rankGauss: ", train_df.shape,test_df.shape)
 train_df,test_df = add_features(train_df,test_df)
 print("add stats feature:", train_df.shape,test_df.shape)
 return train_df,test_df,targets<groupby>
models = model_results['model']
def make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH):
vc = train.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in range(SEEDS):
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.loc[train.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 train[kfold_col] = train.drug_id.map(dct1)
 train.loc[train[kfold_col].isna() , kfold_col] = train.loc[train[kfold_col].isna() , 'sig_id'].map(dct2)
 train[kfold_col] = train[kfold_col].astype('int8')
 return train<init_hyperparams>
test_ids = list(final.loc[final['Target'].isnull() ,'idhogar'] )
no_ctl = True
scale = 'rankgauss'
variance_threshould = 0.8
decompo = 'PCA'
ncompo_genes = 50
ncompo_cells = 8
encoding = 'dummy'
base_seed = 42
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 print(DEVICE)
 EPOCHS = 20
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = True
hidden_size=1024
SEEDS = 7
SEED = [0, 1, 2, 3, 4, 5, 6]
DRUG_THRESH = 18<load_from_csv>
def submit(model,train,train_labels,test,test_ids):
model.fit(train,train_labels)
 predictions = model.predict(test)
 predictions = pd.DataFrame({'idhogar':test_ids,'Target':predictions})
 submission = submission_base.merge(predictions,on='idhogar',how='left' ).drop(columns=['idhogar'])
 submission['Target'] = submission['Target'].fillna(4 ).astype(np.int8)
 return submission
seed_everything(base_seed)
 def preprocessor_Multi_heads() :
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 train_drug = pd.read_csv('.. /input/lish-moa/train_drug.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 train, test, targets = data_processing(train_features,test_features, train_targets_scored)
 print('length of target_cols:',len(target_cols))
train_comb = train.merge(targets, on='sig_id')
 train_comb.drop('cp_type', axis=1, inplace=True)
 print("combine train and targets to prepare for kfolds ", train_comb.shape)
 feature_cols = [c for c in process_data(train_comb ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id', 'drug_id']]
print('length of feature_cols:',len(feature_cols))
gen_cols = [c for c in feature_cols if 'g-' == c[:2]]
cell_cols = [c for c in feature_cols if 'c-' == c[:2]]
pca_gen_cols = [c for c in feature_cols if 'pca_g' == c[:5]]
pca_cell_cols = [c for c in feature_cols if 'pca_c' == c[:5]]
ohe_cols = [c for c in feature_cols if c not in gen_cols+cell_cols+pca_gen_cols+pca_cell_cols]
print(len(gen_cols), len(cell_cols), len(pca_gen_cols), len(pca_cell_cols),len(ohe_cols))
num_features=len(feature_cols)
 num_targets=len(target_cols)
 folds = train_comb.copy()
folds = folds.merge(train_drug, on='sig_id')
 folds = make_cv_folds(folds, SEEDS, NFOLDS, DRUG_THRESH)
 return folds, test, targets,gen_cols, pca_gen_cols, ohe_cols, cell_cols, pca_cell_cols,test_features
folds, test, targets,gen_cols, pca_gen_cols, ohe_cols, cell_cols, pca_cell_cols, test_features = preprocessor_Multi_heads()<choose_model_class>
rf_submission = submit(RandomForestClassifier(n_estimators = 100,
random_state=10, n_jobs = -1),
train_set, train_labels, test_set, test_ids)
 rf_submission.to_csv('rf_submission.csv', index = False )
class Model(nn.Module):
def __init__(self, num_gen_features, num_cell_features,num_targets, hidden_size):
super(Model, self ).__init__()
self.gen_batch_norm1 = nn.BatchNorm1d(num_gen_features)
 self.gen_dropout1 = nn.Dropout(0.2)
 self.gen_dense1 = nn.utils.weight_norm(nn.Linear(num_gen_features, hidden_size))
self.gen_batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.gen_dropout2 = nn.Dropout(0.3)
 self.gen_dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.cell_batch_norm1 = nn.BatchNorm1d(num_cell_features)
 self.cell_dropout1 = nn.Dropout(0.2)
 self.cell_dense1 = nn.utils.weight_norm(nn.Linear(num_cell_features, int(hidden_size/2)))
 self.cell_batch_norm2 = nn.BatchNorm1d(int(hidden_size/2))
self.cell_dropout2 = nn.Dropout(0.3)
 self.cell_dense2 = nn.utils.weight_norm(nn.Linear(int(hidden_size/2), int(hidden_size/2)))
 self.batch_norm3 = nn.BatchNorm1d(hidden_size + int(hidden_size/2))
self.dropout3 = nn.Dropout(0.25)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size + int(hidden_size/2), num_targets))
def forward(self, gen_x, cell_x):
gen_x = self.gen_batch_norm1(gen_x)
 gen_x = self.gen_dropout1(gen_x)
 gen_x = F.relu(self.gen_dense1(gen_x))
gen_x = self.gen_batch_norm2(gen_x)
 gen_x = self.gen_dropout2(gen_x)
 gen_x = F.relu(self.gen_dense2(gen_x))
cell_x = self.cell_batch_norm1(cell_x)
 cell_x = self.cell_dropout1(cell_x)
 cell_x = F.relu(self.cell_dense1(cell_x))
cell_x = self.cell_batch_norm2(cell_x)
 cell_x = self.cell_dropout2(cell_x)
 cell_x = F.relu(self.cell_dense2(cell_x))
x = torch.cat(( gen_x,cell_x),dim=1)
 x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x<categorify>
train_set = pd.DataFrame(train_set,columns=features )
class MoADataset:
def __init__(self, gen_features, cell_features, targets):
self.gen_features = gen_features
self.cell_features = cell_features
self.targets = targets
def __len__(self):
return(self.gen_features.shape[0])
 def __getitem__(self, idx):
dct = {
'gen_x' : torch.tensor(self.gen_features[idx, :], dtype=torch.float),
'cell_x' : torch.tensor(self.cell_features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, gen_features, cell_features):
self.gen_features = gen_features
self.cell_features = cell_features
def __len__(self):
return(self.gen_features.shape[0])
 def __getitem__(self, idx):
dct = {
'gen_x' : torch.tensor(self.gen_features[idx, :], dtype=torch.float),
'cell_x' : torch.tensor(self.cell_features[idx, :], dtype=torch.float)
 }
return dct 
<train_model>
corr_matrix = train_set.corr()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape),k=1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(abs(upper[column])>0.95)]
to_drop
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
gen_x, cell_x, targets = data['gen_x'].to(device), data['cell_x'].to(device), data['y'].to(device)
 outputs = model(gen_x, cell_x)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
gen_x, cell_x, targets = data['gen_x'].to(device), data['cell_x'].to(device), data['y'].to(device)
 outputs = model(gen_x, cell_x)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
gen_x, cell_x = data['gen_x'].to(device), data['cell_x'].to(device)
 with torch.no_grad() :
outputs = model(gen_x, cell_x)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds<normalization>
test_set = pd.DataFrame(test_set,columns=features)
 print(test_set.shape)
 train_set, test_set = train_set.align(test_set,axis=1,join='inner')
 features = list(train_set.columns)
 print(test_set.shape )
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss<prepare_x_and_y>
estimator = RandomForestClassifier(random_state=10,n_estimators=100,n_jobs=-1)
 selector = RFECV(estimator,step=1,cv=5,scoring=scorer,n_jobs=-1 )
def get_oof_predictions(fold,seed):
seed_everything(seed)
 test_ = process_data(test)
 model = Model(
 num_gen_features=len(gen_cols + pca_gen_cols + ohe_cols),
num_cell_features=len(cell_cols + pca_cell_cols + ohe_cols),
num_targets=len(target_cols),
hidden_size=hidden_size)
 model.to(DEVICE)
 model.load_state_dict(torch.load(f".. /input/jp-multiheads/FOLD{fold}_SEED{seed}.pth"))
gen_x_test, cell_x_test = test_[gen_cols + pca_gen_cols + ohe_cols].values, test_[cell_cols + pca_cell_cols + ohe_cols].values
testdataset = TestDataset(gen_x_test, cell_x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 print('predict on test...', end='')
 predictions = np.zeros(( len(test_), targets.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 print(' done.
 ')
 return predictions<categorify>
selector.fit(train_set,train_labels )
def run_k_fold(NFOLDS, seed):
predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
pred_ = get_oof_predictions(fold, seed)
 predictions += pred_ / NFOLDS
return predictions<categorify>
rankings = pd.DataFrame({'feature':list(train_set.columns),'rank':list(selector.ranking_)} ).sort_values(by='rank')
 rankings.head(10 )
def InferenceMultiHeads(SEED):
predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
predictions_ = run_k_fold(NFOLDS, seed)
 predictions += predictions_ / len(SEED)
 print(predictions.shape)
 return predictions
predictions = InferenceMultiHeads(SEED )<create_dataframe>
train_selected = selector.transform(train_set)
 test_selected = selector.transform(test_set )
predictions_clip = predictions
predictions_clip = pd.DataFrame(predictions_clip, columns=target_cols )<load_pretrained>
selected_features = train_set.columns[np.where(selector.ranking_==1)]
train_selected = pd.DataFrame(train_selected,columns=selected_features)
 test_selected = pd.DataFrame(test_selected,columns=selected_features )
oof_multi_heads_all = np.load('.. /input/jp-multiheads/multi_heads_oof.npy' )<save_to_csv>
model_results = cv_model(train_selected,train_labels,model,'RF-SEL',model_results )
test = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sig_id = test[test['cp_type']!='ctl_vehicle'].sig_id.reset_index(drop=True)
 predictions_clip['sig_id'] = sig_id
sub_mh = pd.merge(test[['sig_id']], predictions_clip, on='sig_id', how='left')
 sub_mh.fillna(0, inplace=True)
 sub_mh.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0
sub_mh.to_csv('submission_mh.csv', index=False )<create_dataframe>
def macro_f1_score(labels, predictions):
predictions = predictions.reshape(len(np.unique(labels)) , -1 ).argmax(axis = 0)
 metric_value = f1_score(labels, predictions, average = 'macro')
 return 'macro_f1', metric_value, True
%who DataFrame
del folds, predictions_clip,targets, test,gen_cols, pca_gen_cols, ohe_cols, cell_cols, pca_cell_cols
gc.collect()
%who DataFrame<compute_train_metric>
def log_loss_numpy(y_pred):
y_true_ravel = np.asarray(y_true ).ravel()
y_pred = np.asarray(y_pred ).ravel()
y_pred = np.clip(y_pred, 1e-15, 1 - 1e-15)
 loss = np.where(y_true_ravel == 1, - np.log(y_pred), - np.log(1 - y_pred))
return loss.mean()
def func_numpy_metric(weights):
oof_blend = np.tensordot(weights, oof, axes =(( 0),(0)))
 return log_loss_numpy(oof_blend)
 def grad_func(weights):
oof_clip = np.clip(oof, 1e-15, 1 - 1e-15)
 gradients = np.zeros(oof.shape[0])
 for i in range(oof.shape[0]):
a, b, c = y_true, oof_clip[i], np.zeros(( oof.shape[1], oof.shape[2]))
for j in range(oof.shape[0]):
if j != i:
c += weights[j] * oof_clip[j]
gradients[i] = -np.mean(( -a*b+(b**2)*weights[i]+b*c)/(( b**2)*(weights[i]**2)+2*b*c*weights[i]-b*weights[i]+(c**2)-c))
return gradients
@njit
def grad_func_jit(weights):
oof_clip = np.minimum(1 - 1e-15, np.maximum(oof, 1e-15))
gradients = np.zeros(oof.shape[0])
 for i in range(oof.shape[0]):
a, b, c = y_true, oof_clip[i], np.zeros(( oof.shape[1], oof.shape[2]))
for j in range(oof.shape[0]):
if j != i:
c += weights[j] * oof_clip[j]
gradients[i] = -np.mean(( -a*b+(b**2)*weights[i]+b*c)/(( b**2)*(weights[i]**2)+2*b*c*weights[i]-b*weights[i]+(c**2)-c))
return gradients<load_from_csv>
def model_gbm(features, labels, test_features, test_ids,
nfolds = 5, return_preds = False, hyp = None):

 feature_names = list(features.columns)
 if hyp is not None:
if 'n_estimators' in hyp:
del hyp['n_estimators']
params = hyp
else:
params = {'boosting_type': 'dart',
'colsample_bytree': 0.88,
'learning_rate': 0.028,
'min_child_samples': 10,
'num_leaves': 36, 'reg_alpha': 0.76,
'reg_lambda': 0.43,
'subsample_for_bin': 40000,
'subsample': 0.54,
'class_weight': 'balanced'}
model = lgb.LGBMClassifier(**params, objective = 'multiclass',
n_jobs = -1, n_estimators = 10000,
random_state = 10)
 strkfold = StratifiedKFold(n_splits = nfolds, shuffle = True)
 predictions = pd.DataFrame()
importances = np.zeros(len(feature_names))
features = np.array(features)
 test_features = np.array(test_features)
 labels = np.array(labels ).reshape(( -1))
valid_scores = []
for i,(train_indices, valid_indices)in enumerate(strkfold.split(features, labels)) :
fold_predictions = pd.DataFrame()
X_train = features[train_indices]
X_valid = features[valid_indices]
y_train = labels[train_indices]
y_valid = labels[valid_indices]
model.fit(X_train, y_train, early_stopping_rounds = 100,
eval_metric = macro_f1_score,
eval_set = [(X_train, y_train),(X_valid, y_valid)],
eval_names = ['train', 'valid'],
verbose = 200)
 valid_scores.append(model.best_score_['valid']['macro_f1'])
 fold_probabilitites = model.predict_proba(test_features)
 for j in range(4):
fold_predictions[(j + 1)] = fold_probabilitites[:, j]
fold_predictions['idhogar'] = test_ids
fold_predictions['fold'] =(i+1)
 predictions = predictions.append(fold_predictions)
 importances += model.feature_importances_ / nfolds
display(f'Fold {i + 1}, Validation Score: {round(valid_scores[i], 5)}, Estimators Trained: {model.best_iteration_}')
 feature_importances = pd.DataFrame({'feature': feature_names,
'importance': importances})
 valid_scores = np.array(valid_scores)
 display(f'{nfolds} cross validation score: {round(valid_scores.mean() , 5)} with std: {round(valid_scores.std() , 5)}.')
 if return_preds:
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 return predictions, feature_importances
predictions = predictions.groupby('idhogar', as_index = False ).mean()
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 predictions = predictions.drop(columns = ['fold'])
 submission = submission_base.merge(predictions[['idhogar', 'Target']], on = 'idhogar', how = 'left' ).drop(columns = ['idhogar'])
 submission['Target'] = submission['Target'].fillna(4 ).astype(np.int8)
 return submission, feature_importances, valid_scores
y_true = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv', index_col = 'sig_id' ).values
oof_dict = {'NN-Transfer-Learning': '.. /input/nn-transferlearning-7seeds4248-10folds/oof_nn_transfer_all.npy',
'TabNet': '.. /input/tabnetfe7seeds424810folds/oof_TabNet_all.npy',
'ResNet': '.. /input/resent-1841/resnet_oof.npy',
'Multi-Heads': '.. /input/jp-multiheads/multi_heads_oof.npy'
}
sub = {}
sub['NN-Transfer-Learning']= sub_nn_transfer
sub['TabNet'] = submission_TabNet
sub['ResNet'] = sub_rs
sub['Multi-Heads'] = sub_mh
oof = []
sub_all = []
oof_tmp = np.zeros([y_true.shape[0], y_true.shape[1]])
 for i in oof_dict:
sub_tmp = sub[i].iloc[:,1:].values
if sub_tmp.sum() > 0.0000000001:
oof.append(np.load(oof_dict[i]))
sub_all.append(sub_tmp)
 oof = np.asarray(oof)
 sub_all = np.asarray(sub_all)
<compute_test_metric>
predictions, gbm_fi = model_gbm(train_set,train_labels,test_set,test_ids,return_preds=True )
%%time
log_loss_scores = {}
for n, key in enumerate(oof_dict.keys()):
score_oof = log_loss_numpy(oof[n])
 log_loss_scores[key] = score_oof
print(f'{key} CV:\t', score_oof)
 print('-' * 50 )<define_variables>
predictions.loc[predictions['idhogar'] == '000a08204',:]
test_weights = np.array([1 / oof.shape[0]] * oof.shape[0] )<train_on_grid>
predictions = predictions.groupby('idhogar', as_index = False ).mean()
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 predictions = predictions.drop(columns = ['fold'] )
tol = 1e-10
init_guess = [1 / oof.shape[0]] * oof.shape[0]
bnds = [(0, 1)for _ in range(oof.shape[0])]
cons = {'type': 'eq',
'fun': lambda x: np.sum(x)- 1,
'jac': lambda x: [1] * len(x)}
print('Inital Blend OOF:', func_numpy_metric(init_guess))
start_time = time()
res_scipy = minimize(fun = func_numpy_metric,
x0 = init_guess,
method = 'SLSQP',
jac = grad_func_jit,
bounds = bnds,
constraints = cons,
tol = tol)
 print(f'[{str(datetime.timedelta(seconds = time() - start_time)) [2:7]}] Optimised Blend OOF:', res_scipy.fun)
 print('Optimised Weights:', res_scipy.x )<normalization>
predictions.loc[predictions['idhogar'] == '000a08204',:]
print('Check the sum of all weights:', np.sum(res_scipy.x))
if np.sum(res_scipy.x)- 1 <= tol:
print('Great! The sum of all weights equals to 1!')
 else:
print('Manual adjustion is needed to modify the weights.' )<define_variables>
%%capture
submission, gbm_fi, valid_scores = model_gbm(train_set, train_labels,
test_set, test_ids, return_preds=False)
 submission.to_csv('gbm_baseline.csv' )
lb_weight = []<load_from_csv>
submission, gbm_fi_selected, valid_scores_selected = model_gbm(train_selected, train_labels,
test_selected, test_ids )
sub = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sub.iloc[:,1:] = np.clip(0.25 *sub_nn_transfer.iloc[:, 1:].values + \
0.25 * submission_TabNet.iloc[:,1:].values + \
0.25 * sub_rs.iloc[:,1:].values + \
0.25 * sub_mh.iloc[:,1:].values, 0.00005,0.99995)
 sub.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0
sub.to_csv('submission.csv', index=False )<load_from_csv>
model_results = model_results.append(pd.DataFrame({'model': ["GBM", "GBM_SEL"],
'cv_mean': [valid_scores.mean() , valid_scores_selected.mean() ],
'cv_std': [valid_scores.std() , valid_scores_selected.std() ]}),
sort = True)
 model_results.reset_index(drop=True )
!rm submission_TabNet.csv<set_options>
submission, gbm_fi, valid_scores = model_gbm(train_set, train_labels, test_set, test_ids,
nfolds=10, return_preds=False )
warnings.filterwarnings('ignore' )<install_modules>
submission.to_csv('gbm_10fold.csv', index = False )
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/ 
<load_from_csv>
submission, gbm_fi_selected, valid_scores_selected = model_gbm(train_selected, train_labels, test_selected, test_ids,
nfolds=10 )
train_features = pd.read_csv(data_dir + 'train_features.csv')
 train_targets_scored = pd.read_csv(data_dir + 'train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv(data_dir + 'train_targets_nonscored.csv')
 train_drug = pd.read_csv(data_dir + 'train_drug.csv')
 test_features = pd.read_csv(data_dir + 'test_features.csv')
 sample_submission = pd.read_csv(data_dir + 'sample_submission.csv')
 print('train_features: {}'.format(train_features.shape))
print('train_targets_scored: {}'.format(train_targets_scored.shape))
print('train_targets_nonscored: {}'.format(train_targets_nonscored.shape))
print('train_drug: {}'.format(train_drug.shape))
print('test_features: {}'.format(test_features.shape))
print('sample_submission: {}'.format(sample_submission.shape))<create_dataframe>
submission.to_csv('gmb_10fold_selected.csv', index = False )
train_features2=train_features.copy()
test_features2=test_features.copy()<define_variables>
model_results = model_results.append(pd.DataFrame({'model': ["GBM_10Fold", "GBM_10Fold_SEL"],
'cv_mean': [valid_scores.mean() , valid_scores_selected.mean() ],
'cv_std': [valid_scores.std() , valid_scores_selected.std() ]}),
sort = True )
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<create_dataframe>
print(f"There are {gbm_fi_selected[gbm_fi_selected['importance'] == 0].shape[0]} features with no importance." )
qt = QuantileTransformer(n_quantiles=100,random_state=42,output_distribution='normal')
 data = pd.concat([pd.DataFrame(train_features[GENES+CELLS]), pd.DataFrame(test_features[GENES+CELLS])])
 data2 = qt.fit_transform(data[GENES+CELLS])
 train_features[GENES+CELLS] = pd.DataFrame(data2[:train_features.shape[0]])
 test_features[GENES+CELLS] = pd.DataFrame(data2[-test_features.shape[0]:] )<set_options>
from hyperopt import hp, tpe, Trials, fmin, STATUS_OK
from hyperopt.pyll.stochastic import sample
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(42 )<create_dataframe>
import csv
import ast
from timeit import default_timer as timer
n_comp = 600
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]];
test2 = data2[-test_features.shape[0]:]
train_gpca = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test_gpca = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_gpca), axis=1)
 test_features = pd.concat(( test_features, test_gpca), axis=1 )<create_dataframe>
def objective(hyperparameters,nfolds=5):
global ITERATION
ITERATION += 1
subsample = hyperparameters['boosting_type'].get('subsample',1.0)
 subsample_freq = hyperparameters['boosting_type'].get('subsample_freq',0)
 boosting_type = hyperparameters['boosting_type']['boosting_type']
if boosting_type == 'dart':
hyperparameters['drop_rate'] = hyperparameters['boosting_type']['drop_rate']
hyperparameters['subsample'] = subsample
hyperparameters['subsample_freq'] = subsample_freq
hyperparameters['boosting_type'] = boosting_type
if not hyperparameters['limit_max_depth']:
hyperparameters['max_depth'] = -1
for parameter_name in ['max_depth','num_leaves','subsample_for_bin','min_child_samples','subsample_freq']:
hyperparameters[parameter_name] = int(hyperparameters[parameter_name])
 if 'n_estimators' in hyperparameters:
del hyperparameters['n_estimators']
strkfold = StratifiedKFold(n_splits=nfolds, shuffle=True)
 features = np.array(train_selected)
 labels = np.array(train_labels ).reshape(( -1))
valid_scores = []
best_estimators = []
run_times =[]
model = lgb.LGBMClassifier(**hyperparameters,class_weight='balanced',n_jobs=-1,metric='None',n_estimators=10000)
 for i,(train_indices,valid_indices)in enumerate(strkfold.split(features,labels)) :
X_train,X_valid = features[train_indices],features[valid_indices]
y_train,y_valid = labels[train_indices],labels[valid_indices]
start = timer()
model.fit(X_train,y_train,early_stopping_rounds=100,eval_metric=macro_f1_score,eval_set=[(X_train,y_train),(X_valid,y_valid)],eval_names=['train','valid'],verbose=400)
 end = timer()
valid_scores.append(model.best_score_['valid']['macro_f1'])
 best_estimators.append(model.best_iteration_)
 run_times.append(end-start)
 score = np.mean(valid_scores)
 score_std = np.std(valid_scores)
 loss = 1 - score
run_time = np.mean(run_times)
 run_time_std = np.std(run_times)
 estimators = int(np.mean(best_estimators))
hyperparameters['n_estimators'] = estimators
of_connection = open(OUT_FILE,'a')
 writer = csv.writer(of_connection)
 writer.writerow([loss,hyperparameters,ITERATION,run_time,score,score_std])
 of_connection.close()
if ITERATION % PROGRESS == 0:
display(f'Iteration: {ITERATION}, Current Score: {round(score, 4)}.')
 return {'loss': loss, 'hyperparameters': hyperparameters, 'iteration': ITERATION,
'time': run_time, 'time_std': run_time_std, 'status': STATUS_OK,
'score': score, 'score_std': score_std}
n_comp = 50
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train_cpca = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test_cpca = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_cpca), axis=1)
 test_features = pd.concat(( test_features, test_cpca), axis=1 )<create_dataframe>
space = {
'boosting_type': hp.choice('boosting_type',
[{'boosting_type': 'gbdt',
'subsample': hp.uniform('gdbt_subsample', 0.5, 1),
'subsample_freq': hp.quniform('gbdt_subsample_freq', 1, 10, 1)},
{'boosting_type': 'dart',
'subsample': hp.uniform('dart_subsample', 0.5, 1),
'subsample_freq': hp.quniform('dart_subsample_freq', 1, 10, 1),
'drop_rate': hp.uniform('dart_drop_rate', 0.1, 0.5)},
{'boosting_type': 'goss',
'subsample': 1.0,
'subsample_freq': 0}]),
'limit_max_depth': hp.choice('limit_max_depth', [True, False]),
'max_depth': hp.quniform('max_depth', 1, 40, 1),
'num_leaves': hp.quniform('num_leaves', 3, 50, 1),
'learning_rate': hp.loguniform('learning_rate',
np.log(0.025),
np.log(0.25)) ,
'subsample_for_bin': hp.quniform('subsample_for_bin', 2000, 100000, 2000),
'min_child_samples': hp.quniform('min_child_samples', 5, 80, 5),
'reg_alpha': hp.uniform('reg_alpha', 0.0, 1.0),
'reg_lambda': hp.uniform('reg_lambda', 0.0, 1.0),
'colsample_bytree': hp.uniform('colsample_by_tree', 0.5, 1.0)
 }
var_thresh = VarianceThreshold(0.85)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
<concatenate>
algo = tpe.suggest
def fe_cluster(train, test, n_clusters_g = 22, n_clusters_c = 4, SEED = 42):
features_g = GENES
features_c = CELLS
def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_{kind}'] = kmeans.labels_[:train.shape[0]]
test[f'clusters_{kind}'] = kmeans.labels_[train.shape[0]:]
train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features2 ,test_features2=fe_cluster(train_features2,test_features2 )<concatenate>
trials = Trials()
OUT_FILE = 'optimization.csv'
of_connection = open(OUT_FILE,'w')
 writer = csv.writer(of_connection)
 MAX_EVALS = 100
PROGRESS = 10
N_FOLDS = 5
ITERATION = 0
headers = ['loss', 'hyperparameters', 'iteration', 'runtime', 'score', 'std']
writer.writerow(headers)
 of_connection.close()
train_pca=pd.concat(( train_gpca,train_cpca),axis=1)
 test_pca=pd.concat(( test_gpca,test_cpca),axis=1 )<categorify>
display("Running Optimization for {} Trials.".format(MAX_EVALS))
best = fmin(fn = objective, space = space, algo = tpe.suggest, trials = trials,
max_evals = MAX_EVALS )
def fe_cluster_pca(train, test,n_clusters=5,SEED = 42):
data=pd.concat([train,test],axis=0)
 kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_pca'] = kmeans.labels_[:train.shape[0]]
test[f'clusters_pca'] = kmeans.labels_[train.shape[0]:]
train = pd.get_dummies(train, columns = [f'clusters_pca'])
 test = pd.get_dummies(test, columns = [f'clusters_pca'])
 return train, test
train_cluster_pca ,test_cluster_pca = fe_cluster_pca(train_pca,test_pca )<split>
with open('trials.json', 'w')as f:
f.write(json.dumps(str(trials)) )
train_cluster_pca = train_cluster_pca.iloc[:,650:]
test_cluster_pca = test_cluster_pca.iloc[:,650:]<split>
best_hyp = ast.literal_eval(results.loc[0, 'hyperparameters'])
 best_hyp
train_features_cluster=train_features2.iloc[:,876:]
test_features_cluster=test_features2.iloc[:,876:]<define_variables>
submission, gbm_fi, valid_scores = model_gbm(train_selected, train_labels,
test_selected, test_ids,
nfolds = 10, return_preds=False)
 model_results = model_results.append(pd.DataFrame({'model': ["GBM_OPT_10Fold_SEL"],
'cv_mean': [valid_scores.mean() ],
'cv_std': [valid_scores.std() ]}),
sort = True ).sort_values('cv_mean', ascending = False )
gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577','g-153','g-389','g-60','g-370','g-248','g-167','g-203','g-177','g-301','g-332','g-517','g-6','g-744','g-224','g-162','g-3','g-736','g-486','g-283','g-22','g-359','g-361','g-440','g-335','g-106','g-307','g-745','g-146','g-416','g-298','g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']<feature_engineering>
submission, gbm_fi, valid_scores = model_gbm(train_set, train_labels,
test_set, test_ids,
nfolds = 10, return_preds=False)
 model_results = model_results.append(pd.DataFrame({'model': ["GBM_OPT_10Fold"],
'cv_mean': [valid_scores.mean() ],
'cv_std': [valid_scores.std() ]}),
sort = True ).sort_values('cv_mean', ascending = False )
def fe_stats(train, test):
features_g = GENES
features_c = CELLS
for df in train, test:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-23'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
for feature in features_c:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in gsquarecols:
df[f'{feature}_squared'] = df[feature] ** 2
return train, test
train_features2,test_features2=fe_stats(train_features2,test_features2 )<split>
submission, gbm_fi, valid_scores = model_gbm(train_selected, train_labels,
test_selected, test_ids,
nfolds = 10, return_preds=False )
train_features_stats=train_features2.iloc[:,902:]
test_features_stats=test_features2.iloc[:,902:] 
<concatenate>
submission.to_csv('gbm_opt_10fold_selected.csv', index = False )
train_features = pd.concat(( train_features, train_features_cluster,train_cluster_pca,train_features_stats), axis=1)
 test_features = pd.concat(( test_features, test_features_cluster,test_cluster_pca,test_features_stats), axis=1 )<merge>
heads['Target'].value_counts()
<drop_column><EOS> 
preds['Target'].value_counts()