Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<feature_engineering>
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
df_test['Province_State']=df_test['Province_State'].fillna('')
 df_test['Province_State'].isna().sum()<feature_engineering>
x_train = pd.read_csv('.. /input/lish-moa/train_features.csv')
 x_test = pd.read_csv('.. /input/lish-moa/test_features.csv')
 y_train = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 y_train_non = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
df_test['Country_Region']=df_test['Country_Region']+'_'+df_test['Province_State']<feature_engineering>
trt_cp_index=x_train[x_train['cp_type']=='trt_cp'].index
def create_features(df,label=None):

 df = df.copy()
df['Date'] = df.index
df['dayofweek'] = df['Date'].dt.dayofweek
df['quarter'] = df['Date'].dt.quarter
df['month'] = df['Date'].dt.month
df['dayofyear'] = df['Date'].dt.dayofyear
df['dayofmonth'] = df['Date'].dt.day
df['weekofyear'] = df['Date'].dt.weekofyear
X = df[['dayofweek','quarter','month',
'dayofyear','dayofmonth','weekofyear']]
return X<create_dataframe>
x_test_checkpoint=x_test.copy()
train = df_train.set_index(['Date'])
 train.index = pd.to_datetime(train.index)
 train_features=pd.DataFrame(create_features(train))
train_features<concatenate>
x_train=x_train.drop(['sig_id'],1)
 x_test=x_test.drop(['sig_id'],1)
 y_train=y_train.drop(['sig_id'],1)
 x_train.head()
df_train_final = pd.concat([train,train_features], axis=1)
 df_train_final.reset_index(drop=True ,inplace =True)
 df_train_final<categorify>
x_train=x_train.drop(['cp_type'],1)
 x_test=x_test.drop(['cp_type'],1 )
le = LabelEncoder()
df_train_final['Country_Region'] = le.fit_transform(df_train_final['Country_Region'] )<drop_column>
x_train['cp_time'] = x_train['cp_time'].map({24:0,48:1,72:2})
 x_train.cp_time.unique()
x_test['cp_time'] = x_test['cp_time'].map({24:0,48:1,72:2})
 x_test.cp_time.unique()
df_train_final.drop(columns='Province_State' , inplace=True)
 df_train_final<create_dataframe>
genes=[col for col in x_train.columns if col.startswith("g-")]
cells=[col for col in x_train.columns if col.startswith("c-")]
test = df_test.set_index(['Date'])
 test.index = pd.to_datetime(test.index)
 test_features=pd.DataFrame(create_features(test))
test_features<concatenate>
a_train=x_train.copy()
a_test=x_test.copy()
df_test_final = pd.concat([test,test_features], axis=1)
 df_test_final.reset_index(drop=True ,inplace =True)
 df_test_final<categorify>
x_train=a_train.copy()
x_test=a_test.copy()
le = LabelEncoder()
df_test_final['Country_Region'] = le.fit_transform(df_test_final['Country_Region'] )<drop_column>
scaler = QuantileTransformer(n_quantiles=100,random_state=42, output_distribution="normal")
 scaled_data=scaler.fit_transform(x_train[genes+cells])
 x_train=pd.concat([x_train.iloc[:,:2],pd.DataFrame(scaled_data,columns=genes+cells)],1)
 transformer = QuantileTransformer(n_quantiles=100,random_state=42, output_distribution="normal")
 scaled_data=scaler.fit_transform(x_test[genes+cells])
 x_test=pd.concat([x_test.iloc[:,:2],pd.DataFrame(scaled_data,columns=genes+cells)],1 )
df_test_final.drop(columns='Province_State' , inplace=True)
 df_test_final<drop_column>
data=pd.concat(( x_train[genes],x_test[genes]))
g_data=PCA(n_components=0.95,random_state=44 ).fit_transform(data)
 g_train=g_data[:x_train.shape[0]]
g_test=g_data[x_train.shape[0]:]
g_train_df=pd.DataFrame(g_train,columns=[f'pca_g-{i}' for i in range(g_data.shape[1])])
 g_test_df=pd.DataFrame(g_test,columns=[f'pca_g-{i}' for i in range(g_data.shape[1])])
 data=pd.concat(( x_train[cells],x_test[cells]))
c_data=PCA(n_components=0.95,random_state=44 ).fit_transform(data)
 c_train=c_data[:x_train.shape[0]]
c_test=c_data[x_train.shape[0]:]
c_train_df=pd.DataFrame(c_train,columns=[f'pca_c-{i}' for i in range(c_data.shape[1])])
 c_test_df=pd.DataFrame(c_test,columns=[f'pca_c-{i}' for i in range(c_data.shape[1])] )
df_test_final = df_test_final.drop(['ForecastId'],axis=1 )<data_type_conversions>
X_train=pd.concat(( x_train,g_train_df,c_train_df),axis=1)
 X_test=pd.concat(( x_test,g_test_df,c_test_df),axis=1 )
df_train_final['ConfirmedCases'] = df_train_final['ConfirmedCases'].apply(int)
 df_train_final['Fatalities'] = df_train_final['Fatalities'].apply(int )<prepare_x_and_y>
varthre = VarianceThreshold(0.8)
 train_varthre=varthre.fit_transform(X_train.iloc[:,2:])
 test_varthre=varthre.transform(X_test.iloc[:,2:] )
y_conf=df_train_final['ConfirmedCases']
y_conf.sample(6)
 y_fat=df_train_final['Fatalities']<drop_column>
X_train=pd.concat([X_train.iloc[:,:2],pd.DataFrame(train_varthre,columns=X_train.columns[2:][varthre.get_support() ])],1)
 X_test=pd.concat([X_test.iloc[:,:2],pd.DataFrame(test_varthre,columns=X_test.columns[2:][varthre.get_support() ])],1)
 X_train.shape,X_test.shape,y_train.shape,submission.shape
df_train_final_fat = df_train_final.drop(['Id','Fatalities'],axis=1)
 df_train_final_fat.columns<drop_column>
genes=[col for col in X_train.columns if col.startswith("g-")]
cells=[col for col in X_train.columns if col.startswith("c-")]
df_train_final_ConCases = df_train_final.drop(['Id','ConfirmedCases','Fatalities'],axis=1)
 df_train_final_ConCases.columns<split>
kmeans=KMeans(n_clusters=35,random_state=982)
 kmeans.fit(pd.concat([X_train[genes],X_test[genes]]))
X_train["cluster_g"]=kmeans.labels_[:len(X_train)]
X_test["cluster_g"]=kmeans.labels_[len(X_train):]
kmeans=KMeans(n_clusters=5,random_state=982)
 kmeans.fit(pd.concat([X_train[cells],X_test[cells]]))
X_train["cluster_c"]=kmeans.labels_[:len(X_train)]
X_test["cluster_c"]=kmeans.labels_[len(X_train):]
X_train=pd.get_dummies(X_train,columns=['cluster_g','cluster_c'])
 X_test=pd.get_dummies(X_test,columns=['cluster_g','cluster_c'] )
X_train,X_test,y_train,y_test = train_test_split(df_train_final_ConCases,y_conf,test_size=0.2,random_state=42 )<train_model>
gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577','g-153','g-389','g-60','g-370','g-248','g-167','g-203','g-177','g-301','g-332','g-517','g-6','g-744','g-224','g-162','g-3','g-736','g-486','g-283','g-22','g-359','g-361','g-440','g-335','g-106','g-307','g-745','g-146','g-416','g-298','g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']
rf = XGBRegressor(n_estimators = 2400 , random_state = 0 , max_depth = 26)
 rf.fit(X_train,y_train )<predict_on_test>
for df in X_train,X_test:
df['time_dose']=df['cp_time']+df['cp_dose']
df['g_median']=df[genes].median(axis=1)
 df['c_median']=df[cells].median(axis=1)
 df['gc_median']=df[genes+cells].median(axis=1)
 df['g_max']=df[genes].max(axis=1)
 df['c_max']=df[cells].max(axis=1)
 df['gc_max']=df[genes+cells].max(axis=1)
 df['g_min']=df[genes].min(axis=1)
 df['c_min']=df[cells].min(axis=1)
 df['gc_min']=df[genes+cells].min(axis=1)
 df['g_sum']=df[genes].sum(axis=1)
 df['c_sum']=df[cells].sum(axis=1)
 df['gc_sum']=df[genes+cells].sum(axis=1)
 df['g_mean']=df[genes].mean(axis=1)
 df['c_mean']=df[cells].mean(axis=1)
 df['gc_mean']=df[genes+cells].mean(axis=1)
 df['g_std']=df[genes].std(axis=1)
 df['c_std']=df[cells].std(axis=1)
 df['gc_std']=df[genes+cells].std(axis=1)
 df['g_skew']=df[genes].skew(axis=1)
 df['c_skew']=df[cells].skew(axis=1)
 df['gc_skew']=df[genes+cells].skew(axis=1)
 df['g_kurtosis']=df[genes].kurtosis(axis=1)
 df['c_kurtosis']=df[cells].kurtosis(axis=1)
 df['gc_kurtosis']=df[genes+cells].kurtosis(axis=1)
 df['g_q25']=df[genes].quantile(0.25,axis=1)
 df['c_q25']=df[cells].quantile(0.25,axis=1)
 df['gc_q25']=df[genes+cells].quantile(0.25,axis=1)
 df['g_q75']=df[genes].quantile(0.75,axis=1)
 df['c_q75']=df[cells].quantile(0.75,axis=1)
 df['gc_q75']=df[genes+cells].quantile(0.75,axis=1)
 df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-23'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
for feature in cells:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in gsquarecols:
df[f'{feature}_squared'] = df[feature] ** 2
pred_conf = rf.predict(X_test)
 predictions = [round(value)for value in pred_conf]
predictions<compute_test_metric>
X_train=X_train.loc[trt_cp_index,:]
y_train=y_train.loc[trt_cp_index,:]
y_train_non=y_train_non.loc[trt_cp_index,:]
rf.score(X_test,y_test )<compute_test_metric>
hps={
30:[
{
"units":[1410,1246,512,512,512],
"activations":['selu', 'swish', 'elu','elu','elu'],
"dropouts":[0.49,0.51,0.0,0.0,0.0]
},
{
"units":[1127,1133, 1209, 512],
"activations":[ 'selu','swish','swish','elu'],
"dropouts":[0.32,0.02,0.26,0.0]
},
{
"units":[796,1995,573],
"activations":['selu','selu','swish'],
"dropouts":[0.22,0.21,0.09]
},
{
"units":[959,1557,1826],
"activations":['selu','selu','elu'],
"dropouts":[0.42,0.61,0.3]
},
{
"units":[1081,2033],
"activations":['elu','elu'],
"dropouts":[0.44,0.12]
}
],
58:[
{
"units":[561,882,2023],
"activations":['elu','elu','elu'],
"dropouts":[0.43,0.17,0.29]
},
{
"units":[687,1085,1365,1427,1325],
"activations":['selu','swish','swish','elu','selu'],
"dropouts":[0.59,0.15,0.31,0.08,0.41]
},
{
"units":[1324,1475,976,512],
"activations":['selu','selu','selu', 'elu'],
"dropouts":[0.39,0.38,0.19,0.0]
},
{
"units":[1410,1246,512,512,512],
"activations":['selu', 'swish', 'elu', 'elu', 'elu'],
"dropouts":[0.49,0.51,0.0,0.0,0.0]
},
{
"units":[750,1734,638,512],
"activations":['selu','swish','selu','elu'],
"dropouts":[0.42,0.5,0.42,0.0]
}
],
3255:[
{
"units":[1444,629,512,512,512],
"activations":['selu', 'swish', 'elu', 'elu', 'elu'],
"dropouts":[0.62,0.4,0.0,0.0,0.0]
},
{
"units":[899,940,893,718],
"activations":['elu', 'elu', 'selu', 'selu'],
"dropouts":[0.33,0.28,0.29,0.17]
},
{
"units":[2030,1422],
"activations":['elu', 'selu'],
"dropouts":[0.53,0.37]
},
{
"units":[1194,1562],
"activations":[ 'swish','swish'],
"dropouts":[0.59,0.0]
},
{
"units":[1130,766,520,512],
"activations":[ 'selu','elu','elu', 'elu'],
"dropouts":[0.29,0.14,0.27,0.0]
}
]
}
r=np.sqrt(mean_squared_error(y_test,predictions))
r<compute_test_metric>
from pickle import load,dump
mae = mean_absolute_error(y_test,predictions)
 print('MAE: %f' % mae )<compute_test_metric>
y_train_non=y_train_non.iloc[:,y_train_non.columns!='sig_id']
final_y=pd.concat([y_train,y_train_non],1)
 final_pred=pd.DataFrame(np.zeros(( submission.shape[0], final_y.shape[1])) ,columns=final_y.columns)
 for seed in [30,58,3255]:
for n,(tr,te)in enumerate(MultilabelStratifiedKFold(n_splits=5, random_state=seed, shuffle=True ).split(X_train,y_train)) :
print("SEED:"+str(seed)+" Fold:"+str(n))
units=hps[seed][n]['units']
activations=hps[seed][n]['activations']
dropouts=hps[seed][n]['dropouts']
model=create_model(X_train.shape[1], units, activations, dropouts, 1e-3, final_y.shape[1])
 model.fit(X_train.values[tr],
final_y.values[tr],
batch_size=128,
epochs=150,
validation_data=(X_train.values[te],final_y.values[te]),
verbose=1,
callbacks=[
tf.keras.callbacks.ReduceLROnPlateau(
 monitor='val_loss',
factor=0.1,
patience=3,
epsilon = 1e-4,
mode = 'min',
verbose=1 
)
 ,
tf.keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=0,
patience=10,
mode='auto',
verbose=1,
baseline=None,
restore_best_weights=True 
)
 ] 
)
 model.save(f'pSEED{seed}FOLD{n}.h5')
 model=create_model(X_train.shape[1], units, activations, dropouts, 1e-3, final_y.shape[1])
 model.load_weights(f'pSEED{seed}FOLD{n}.h5')
 final_pred+=model.predict(X_test )
mse = mean_squared_error(y_test,predictions)
 print('MSE: %f' % mse )<compute_test_metric>
y_pred=final_pred.loc[:,submission.columns[1:]]/15
rmse = sqrt(mse)
 print('RMSE: %f' % rmse )<compute_test_metric>
final_pred=submission.iloc[:,1:].copy()
final_pred.loc[:,:]=0
R2=r2_score(y_test,predictions)
 print('R2: %f' % R2 )<create_dataframe>
for seed in [30,58,3255]:
for n,(tr,te)in enumerate(MultilabelStratifiedKFold(n_splits=5, random_state=seed, shuffle=True ).split(X_train,y_train)) :
print("SEED:"+str(seed)+" Fold:"+str(n))
units=hps[seed][n]['units']
activations=hps[seed][n]['activations']
dropouts=hps[seed][n]['dropouts']
model2=create_model(X_train.shape[1], units, activations, dropouts, 1e-3,y_train_non.shape[1])
 model2.fit(X_train.values[tr],
y_train_non.values[tr],
batch_size=128,
epochs=150,
validation_data=(X_train.values[te],y_train_non.values[te]),
verbose=1,
callbacks=[
tf.keras.callbacks.ReduceLROnPlateau(
 monitor='val_loss',
factor=0.1,
patience=3,
epsilon = 1e-4,
mode = 'min',
verbose=1 
)
 ,
tf.keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=0,
patience=10,
mode='auto',
verbose=1,
baseline=None,
restore_best_weights=True 
)
 ] 
)
 model=create_model(X_train.shape[1], units, activations, dropouts, 1e-3, y_train.shape[1])
 for i in range(len(model.layers)-1):
model.layers[i].set_weights(model2.layers[i].get_weights())
 model.fit(X_train.values[tr],
y_train.values[tr],
batch_size=128,
epochs=150,
validation_data=(X_train.values[te],y_train.values[te]),
verbose=1,
callbacks=[
tf.keras.callbacks.ReduceLROnPlateau(
 monitor='val_loss',
factor=0.1,
patience=3,
epsilon = 1e-4,
mode = 'min',
verbose=1 
)
 ,
tf.keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=0,
patience=10,
mode='auto',
verbose=1,
baseline=None,
restore_best_weights=True 
)
 ] 
)
 model.save(f'tSEED{seed}FOLD{n}.h5')
 model=create_model(X_train.shape[1], units, activations, dropouts, 1e-3, y_train.shape[1])
 model.load_weights(f'tSEED{seed}FOLD{n}.h5')
 final_pred+=model.predict(X_test )
dfpred = pd.DataFrame(data=predictions, columns=[ "pred"] , index=y_test.index)
 dfpred1=pd.concat([dfpred,y_test],axis=1)
 dfpred1.sample(30 )<split>
y_pred_transfer=final_pred/15
X_train2,X_test2,y_train2,y_test2 = train_test_split(df_train_final_fat,y_fat,test_size=0.2,random_state=42 )<train_model>
class LogitsLogLoss(Metric):

 def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):

 logits = 1 /(1 + np.exp(-y_pred))
aux =(1-y_true)*np.log(1-logits+1e-15)+ y_true*np.log(logits+1e-15)
 return np.mean(-aux )
rf2 = XGBRegressor(n_estimators = 2000 , random_state = 0 , max_depth = 20)
 rf2.fit(X_train2,y_train2 )<predict_on_test>
tabnet_params = dict(
 n_d = 32,
n_a = 32,
n_steps = 1,
gamma = 1.3,
lambda_sparse = 0,
optimizer_fn = optim.Adam,
optimizer_params = dict(lr = 2e-2, weight_decay = 1e-5),
mask_type = "entmax",
scheduler_params = dict(
 mode = "min", patience = 5, min_lr = 1e-5, factor = 0.9),
scheduler_fn = ReduceLROnPlateau,
seed = 456,
verbose = 10
)
pred_fat = rf2.predict(X_test2)
 predictions_fat = [round(value)for value in pred_fat]
predictions_fat<compute_test_metric>
final_pred=submission.iloc[:,1:].copy()
final_pred.loc[:,:]=0
for seed in [30,58,3255]:
for n,(tr,te)in enumerate(MultilabelStratifiedKFold(n_splits=5, random_state=seed, shuffle=True ).split(X_train,y_train)) :
print("SEED:"+str(seed)+" Fold:"+str(n))
model = TabNetRegressor(**tabnet_params)
 model.fit(
 X_train = X_train.values[tr],
y_train = y_train.values[tr],
eval_set = [(X_train.values[te],y_train.values[te])],
eval_name = ["val"],
eval_metric = ["logits_ll"],
max_epochs = 200,
patience = 20,
batch_size = 512,
virtual_batch_size = 32,
num_workers = 1,
drop_last = False,
loss_fn = F.binary_cross_entropy_with_logits)
 model.save_model(f"tabSEED{seed}FOLD{n}")
 final_pred+=1 /(1 + np.exp(-model.predict(X_test.values)) )
rf2.score(X_test2,y_test2 )<compute_test_metric>
y_pred_tabnet=final_pred/15
r=np.sqrt(mean_squared_error(y_test2,predictions_fat))
r<compute_test_metric>
y_pred_final=y_pred * 0.25 + y_pred_transfer * 0.25 + y_pred_tabnet * 0.5
mae2 = mean_absolute_error(y_test2,predictions_fat)
 print('MAE: %f' % mae2 )<compute_test_metric>
submission.iloc[:,1:]=y_pred_final
mse2 = mean_squared_error(y_test2,predictions_fat)
 print('MSE: %f' % mse2 )<compute_test_metric>
submission.loc[x_test_checkpoint["cp_type"]=='ctl_vehicle','5-alpha_reductase_inhibitor':]=0
rmse2 = sqrt(mse2)
 print('RMSE: %f' % rmse2 )<compute_test_metric>
submission.to_csv('submission.csv',index=False )