Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<load_from_csv>
!pip install.. /input/pytorch-tabnet/pytorch_tabnet-2.0.1-py3-none-any.whl
!pip install.. /input/iter-strart/iterative_stratification-0.1.6-py3-none-any.whl
NUM_BRANDS = 2500
NAME_MIN_DF = 10
MAX_FEAT_DESCP = 10000
print("Reading in Data")
 df_train = pd.read_csv('.. /input/train.tsv', sep='\t')
 df_test = pd.read_csv('.. /input/test.tsv', sep='\t')
 df = pd.concat([df_train, df_test], 0)
 nrow_train = df_train.shape[0]
y_train = np.log(df_train["price"]+1)
 del df_train
gc.collect()
print(df.memory_usage(deep = True))
df["category_name"] = df["category_name"].fillna("Other" ).astype("category")
 df["brand_name"] = df["brand_name"].fillna("unknown")
 pop_brands = df["brand_name"].value_counts().index[:NUM_BRANDS]
df.loc[~df["brand_name"].isin(pop_brands), "brand_name"] = "Other"
df["item_description"] = df["item_description"].fillna("None")
 df["item_condition_id"] = df["item_condition_id"].astype("category")
 df["brand_name"] = df["brand_name"].astype("category")
 print(df.memory_usage(deep = True))
print("Encodings")
 count = CountVectorizer(min_df=NAME_MIN_DF)
 X_name = count.fit_transform(df["name"])
 print("Category Encoders")
 unique_categories = pd.Series("/".join(df["category_name"].unique().astype("str")).split("/")).unique()
count_category = CountVectorizer()
X_category = count_category.fit_transform(df["category_name"])
 print("Descp encoders")
 count_descp = TfidfVectorizer(max_features = MAX_FEAT_DESCP,
ngram_range =(1,3),
stop_words = "english")
 X_descp = count_descp.fit_transform(df["item_description"])
 print("Brand encoders")
 vect_brand = LabelBinarizer(sparse_output=True)
 X_brand = vect_brand.fit_transform(df["brand_name"])
 print("Dummy Encoders")
 X_dummies = scipy.sparse.csr_matrix(pd.get_dummies(df[[
"item_condition_id", "shipping"]], sparse = True ).values)
 X = scipy.sparse.hstack(( X_dummies,
X_descp,
X_brand,
X_category,
X_name)).tocsr()
print([X_dummies.shape, X_category.shape,
X_name.shape, X_descp.shape, X_brand.shape])
 X_train = X[:nrow_train]
model = Ridge(solver = "lsqr", fit_intercept=False)
 print("Fitting Model")
 model.fit(X_train, y_train)
 X_test = X[nrow_train:]
preds1 = model.predict(X_test)
 df_test["price1"] = np.exp(preds1)-1 
<compute_test_metric>
warnings.filterwarnings('ignore' )
X=X_train
y=y_train
def rmsle(predictions, targets):
predictions = np.exp(predictions)- 1
targets = np.exp(targets)- 1
return np.sqrt(((predictions - targets)** 2 ).mean())
 def rmsle_lgb(labels, preds):
return 'rmsle', rmsle(preds, labels), False 
<train_model>
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(33 )
print('Training model...')
 X_train, X_test2, y_train, y_test2 = train_test_split(X, y, test_size=0.3, random_state=42)
 lgbm_params = {'n_estimators': 1000, 'learning_rate': 0.4, 'max_depth': 10,
'num_leaves': 31, 'subsample': 0.9, 'colsample_bytree': 0.8,
'min_child_samples': 50, 'n_jobs': 4}
model = LGBMRegressor(**lgbm_params)
 model.fit(X_train, y_train,
eval_set=[(X_test2, y_test2)],
eval_metric=rmsle_lgb,
early_stopping_rounds=100,
verbose=True)
 print('Generating submission...')
 preds2 = model.predict(X_test)
<save_to_csv>
class LogitsLogLoss(Metric):
def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):
logits = 1 /(1 + np.exp(-y_pred))
aux =(1-y_true)*np.log(1-logits+1e-15)+ y_true*np.log(logits+1e-15)
 return np.mean(-aux )
df_test['price2']=np.exp(preds2)- 1
df_test['price']=(df_test['price2']+df_test['price1'])/2
df_test[["test_id", "price"]].to_csv("submission.csv", index = False)
<compute_test_metric>
def load_data(path):
train_feature = pd.read_csv(f'{path}train_features.csv')
 train_target = pd.read_csv(f'{path}train_targets_scored.csv')
 test_feature = pd.read_csv(f'{path}test_features.csv')
 sample_output = pd.read_csv(f'{path}sample_submission.csv')
 features = train_feature.columns[1:]
drug = pd.read_csv(f'{path}train_drug.csv')
 targets = train_target.columns[1:]
train_target = train_target.merge(drug, on='sig_id', how='left')
 return train_feature,train_target,test_feature,sample_output,targets,features
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
return(sum(to_sum)*(1.0/len(y)))** 0.5 
<load_from_csv>
def LabelEncoding(train_feature,test_feature):
le = LabelEncoder()
le1 = LabelEncoder()
le2 = LabelEncoder()
train_feature['cp_type'] = pd.DataFrame(data = le.fit_transform(train_feature['cp_type']))
train_feature['cp_time'] = pd.DataFrame(data = le1.fit_transform(train_feature['cp_time']))
train_feature['cp_dose'] = pd.DataFrame(data = le2.fit_transform(train_feature['cp_dose']))
test_feature['cp_type'] = pd.DataFrame(data = le.transform(test_feature['cp_type']))
test_feature['cp_time'] = pd.DataFrame(data = le1.transform(test_feature['cp_time']))
test_feature['cp_dose'] = pd.DataFrame(data = le2.transform(test_feature['cp_dose']))
return train_feature,test_feature
print("Loading data...")
 train = pd.read_table(".. /input/train.tsv")
 test = pd.read_table(".. /input/test.tsv")
 print(train.shape)
 print(test.shape )<categorify>
def PreProcess(train_feature,test_feature):
GENES = [col for col in train_feature.columns if col.startswith('g-')]
CELLS = [col for col in train_feature.columns if col.startswith('c-')]
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_feature[col].values)
 vec_len_test = len(test_feature[col].values)
 raw_vec = train_feature[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_feature[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_feature[col] = transformer.transform(test_feature[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
n_comp = 600
pca_g = PCA(n_components=n_comp, random_state=42)
 data = pd.concat([pd.DataFrame(train_feature[GENES]), pd.DataFrame(test_feature[GENES])])
 gpca=(pca_g.fit(data[GENES]))
train2=(gpca.transform(train_feature[GENES]))
test2 =(gpca.transform(test_feature[GENES]))
train_gpca = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test_gpca = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_feature = pd.concat(( train_feature, train_gpca), axis=1)
 test_feature = pd.concat(( test_feature, test_gpca), axis=1)
 dump(gpca, open('gpca.pkl', 'wb'))
n_comp = 50
pca_c = PCA(n_components=n_comp, random_state=42)
 data = pd.concat([pd.DataFrame(train_feature[CELLS]), pd.DataFrame(test_feature[CELLS])])
 cpca=(pca_c.fit(data[CELLS]))
train2=(cpca.transform(train_feature[CELLS]))
test2 =(cpca.transform(test_feature[CELLS]))
train_cpca = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test_cpca = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_feature = pd.concat(( train_feature, train_cpca), axis=1)
 test_feature = pd.concat(( test_feature, test_cpca), axis=1)
 dump(cpca, open('cpca.pkl', 'wb'))
c_n = [f for f in list(train_feature.columns)if f not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose']]
mask =(train_feature[c_n].var() >= 0.85 ).values
tmp = train_feature[c_n].loc[:, mask]
train_feature = pd.concat([train_feature[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1)
 tmp = test_feature[c_n].loc[:, mask]
test_feature = pd.concat([test_feature[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1)
 return train_feature,test_feature
print("Handling missing values...")
 def handle_missing(dataset):
dataset.category_name.fillna(value="missing", inplace=True)
 dataset.brand_name.fillna(value="missing", inplace=True)
 dataset.item_description.fillna(value="missing", inplace=True)
 return(dataset)
 train = handle_missing(train)
 test = handle_missing(test)
 print(train.shape)
 print(test.shape )<categorify>
def VarThreshold(train_feature,test_feature):
c_n = [f for f in list(train_feature.columns)if f not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose']]
mask =(train_feature[c_n].var() >= 0.85 ).values
tmp = train_feature[c_n].loc[:, mask]
train_feature = pd.concat([train_feature[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1)
 tmp = test_feature[c_n].loc[:, mask]
test_feature = pd.concat([test_feature[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1)
 return train_feature,test_feature
print("Handling categorical variables...")
 le = LabelEncoder()
le.fit(np.hstack([train.category_name, test.category_name]))
train.category_name = le.transform(train.category_name)
 test.category_name = le.transform(test.category_name)
 le.fit(np.hstack([train.brand_name, test.brand_name]))
train.brand_name = le.transform(train.brand_name)
 test.brand_name = le.transform(test.brand_name)
 del le
train.head(3)
<feature_engineering>
def Stratify_Drugs(scored, threshold, FOLDS, SEED):
vc = scored.drug_id.value_counts()
vc1 = vc.loc[vc<=threshold].index.sort_values()
vc2 = vc.loc[vc>threshold].index.sort_values()
dct1 = {}; dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=FOLDS, shuffle=True,
random_state=SEED)
 tmp = scored.groupby('drug_id')[targets].mean().loc[vc1]
for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[targets])) :
dd = {k:fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=FOLDS, shuffle=True,
random_state=SEED)
 tmp = scored.loc[scored.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[targets])) :
dd = {k:fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 scored['fold'] = scored.drug_id.map(dct1)
 scored.loc[scored.fold.isna() ,'fold'] = scored.loc[scored.fold.isna() ,'sig_id'].map(dct2)
 scored.fold = scored.fold.astype('int8')
 return scored
print("Text to seq process...")
 raw_text = np.hstack([train.item_description.str.lower() , train.name.str.lower() ])
 print(" Fitting tokenizer...")
 tok_raw = Tokenizer()
tok_raw.fit_on_texts(raw_text)
 print(" Transforming text to seq...")
 train["seq_item_description"] = tok_raw.texts_to_sequences(train.item_description.str.lower())
 test["seq_item_description"] = tok_raw.texts_to_sequences(test.item_description.str.lower())
 train["seq_name"] = tok_raw.texts_to_sequences(train.name.str.lower())
 test["seq_name"] = tok_raw.texts_to_sequences(test.name.str.lower())
 train.head(3 )<define_variables>
def Train_Drug(MAX_EPOCH, FOLDS, train_feature,scored,test_feature):
unused_feat = ['sig_id']
features = [ col for col in train_feature.columns if col not in unused_feat]
train_feature = train_feature[features]
test_feature = test_feature[features]
X_test = test_feature.values
tabnet_params = dict(n_d=24, n_a=24, n_steps=1, gamma=1.3,
lambda_sparse=0, optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2, weight_decay=1e-5),
mask_type='entmax',
scheduler_params=dict(mode="min",
patience=5,
min_lr=1e-5,
factor=0.9,),
scheduler_fn=torch.optim.lr_scheduler.ReduceLROnPlateau,
verbose=10, 
)
 test_preds = []
for i in range(FOLDS):
X_train = train_feature.loc[scored.fold != i].values
y_train = scored.loc[scored.fold != i][targets].values
X_val = train_feature.loc[scored.fold == i].values
y_val = scored.loc[scored.fold == i][targets].values
model = TabNetRegressor(**tabnet_params)
 model.fit(X_train=X_train,
y_train=y_train,
eval_set=[(X_val, y_val)],
eval_name = ["val"],
eval_metric = ["logits_ll"],
max_epochs=MAX_EPOCH,
patience=50, batch_size=1024, virtual_batch_size=128,
num_workers=1, drop_last=False,
loss_fn=torch.nn.functional.binary_cross_entropy_with_logits)
 y_test = model.predict(X_test)
 test_preds.append(1 /(1 + np.exp(-y_test)))
 test_preds_total = np.stack(test_preds)
 return test_preds_total
MAX_NAME_SEQ = 10
MAX_ITEM_DESC_SEQ = 75
MAX_TEXT = np.max([np.max(train.seq_name.max())
 , np.max(test.seq_name.max())
 , np.max(train.seq_item_description.max())
 , np.max(test.seq_item_description.max())])+2
MAX_CATEGORY = np.max([train.category_name.max() , test.category_name.max() ])+1
MAX_BRAND = np.max([train.brand_name.max() , test.brand_name.max() ])+1
MAX_CONDITION = np.max([train.item_condition_id.max() , test.item_condition_id.max() ])+1<normalization>
def Train_Simple(MAX_EPOCH,FOLDS,SEED,train_feature,train_target,test_feature):
unused_feat = ['sig_id']
features = [ col for col in train_feature.columns if col not in unused_feat]
train_feature = train_feature[features]
train_target = train_target.drop(["sig_id",'drug_id','fold'], axis=1)
 test_feature = test_feature[features]
X_test = test_feature.values
tabnet_params = dict(n_d=24, n_a=24, n_steps=1, gamma=1.3,
lambda_sparse=0, optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2, weight_decay=1e-5),
mask_type='entmax',
scheduler_params=dict(mode="min",
patience=5,
min_lr=1e-5,
factor=0.9,),
scheduler_fn=torch.optim.lr_scheduler.ReduceLROnPlateau,
verbose=10, 
)
 test_preds = []
mskf = MultilabelStratifiedKFold(n_splits=FOLDS, random_state=SEED, shuffle=True)
 for fold_nb,(train_idx, val_idx)in enumerate(mskf.split(train_feature, train_target)) :
print("FOLDS : ", fold_nb)
 X_train, y_train = train_feature.values[train_idx, :], train_target.values[train_idx, :]
X_val, y_val = train_feature.values[val_idx, :], train_target.values[val_idx, :]
model = TabNetRegressor()
model.fit(X_train=X_train,
y_train=y_train,
eval_set=[(X_val, y_val)],
eval_name = ["val"],
eval_metric = ["logits_ll"],
max_epochs=MAX_EPOCH,
patience=20, batch_size=1024, virtual_batch_size=128,
num_workers=1, drop_last=False,
loss_fn=torch.nn.functional.binary_cross_entropy_with_logits)
 y_test = model.predict(X_test)
 test_preds.append(1 /(1 + np.exp(-y_test)))
 test_preds_total = np.stack(test_preds)
 return test_preds_total
train["target"] = np.log(train.price+1)
 target_scaler = MinMaxScaler(feature_range=(-1, 1))
train["target"] = target_scaler.fit_transform(train.target.reshape(-1,1))
pd.DataFrame(train.target ).hist()<split>
threshold = 19
FOLDS = 5
SEED = 33
epochs = 500
train_feature,train_target,test_feature,sample_output,targets,features = load_data('/kaggle/input/lish-moa/')
 train_feature,test_feature = LabelEncoding(train_feature,test_feature)
 train_feature,test_feature = PreProcess(train_feature,test_feature)
 train_feature,test_feature = VarThreshold(train_feature,test_feature)
 scored = Stratify_Drugs(train_target, threshold, FOLDS, SEED)
 pred = Train_Drug(epochs, FOLDS, train_feature,scored,test_feature)
dtrain, dvalid = train_test_split(train, random_state=123, train_size=0.99)
 print(dtrain.shape)
 print(dvalid.shape )<categorify>
sample_output1 = sample_output.copy()
all_feat = [col for col in sample_output.columns if col not in ["sig_id"]]
sample_output1[all_feat] = pred.mean(axis=0)
 sample_output1.loc[test_feature['cp_type']==0, sample_output.columns[1:]] = 0
sample_output1.to_csv('submission.csv', index=None )