Kaggle competition: Mechanisms of Action (MoA) Prediction
train_data = pd.read_table('.. /input/jerboris/train.tsv')
 test_data = pd.read_table('.. /input/jerboris/test.tsv')
 train_data.head() 
<count_missing_values>
warnings.filterwarnings('ignore')
 BOOK1 = '.. /input/daebook1c/'
BOOK2 = '.. /input/daebook2c/'
CTT = 7
def check_nan(data):
data_check = data.replace('NaN',np.nan)
 data_check = data.replace('No description yet',np.nan)
 data_nan = data_check.isnull().sum()
data_nan.sort_values(ascending=False, inplace=True)
 print('
Total number of NaN values in the dataset:',data_nan.sum())
 print('
Top three features with most NaN values in the dataset:')
 print(data_nan[:5])
 check_nan(train_data)
 check_nan(test_data )<drop_column>
def reduce_mem_usage(props):
start_mem_usg = props.memory_usage().sum() / 1024**2
print("Memory usage of properties dataframe is :",start_mem_usg," MB")
 NAlist = []
for col in props.columns:
if props[col].dtype != object:
IsInt = False
mx = props[col].max()
mn = props[col].min()
if not np.isfinite(props[col] ).all() :
NAlist.append(col)
 props[col].fillna(mn-1,inplace=True)
 asint = props[col].fillna(0 ).astype(np.int64)
 result =(props[col] - asint)
 result = result.sum()
if result > -0.01 and result < 0.01:
IsInt = True
if IsInt:
if mn >= 0:
if mx < 255:
props[col] = props[col].astype(np.uint8)
 elif mx < 65535:
props[col] = props[col].astype(np.uint16)
 elif mx < 4294967295:
props[col] = props[col].astype(np.uint32)
 else:
props[col] = props[col].astype(np.uint64)
 else:
if mn > np.iinfo(np.int8 ).min and mx < np.iinfo(np.int8 ).max:
props[col] = props[col].astype(np.int8)
 elif mn > np.iinfo(np.int16 ).min and mx < np.iinfo(np.int16 ).max:
props[col] = props[col].astype(np.int16)
 elif mn > np.iinfo(np.int32 ).min and mx < np.iinfo(np.int32 ).max:
props[col] = props[col].astype(np.int32)
 elif mn > np.iinfo(np.int64 ).min and mx < np.iinfo(np.int64 ).max:
props[col] = props[col].astype(np.int64)
 else:
props[col] = props[col].astype(np.float32)
 print("___MEMORY USAGE AFTER COMPLETION:___")
 mem_usg = props.memory_usage().sum() / 1024**2
print("Memory usage is: ",mem_usg," MB")
 print("This is ",100*mem_usg/start_mem_usg,"% of the initial size")
 return props, NAlist
def train_short_form_loader(feature_file,target_file,extra_target_file=None):

 train_features = pd.read_csv(feature_file)
 train_targets = pd.read_csv(target_file)
 train_features,_= reduce_mem_usage(train_features)
 train_targets,_ = reduce_mem_usage(train_targets)
 if extra_target_file is not None:
extra_targets = pd.read_csv(extra_target_file)
 extra_targets,_ = reduce_mem_usage(extra_targets)
 train_targets = pd.merge(train_targets,extra_targets,on ='sig_id')
 del extra_targets
targets = train_targets.columns[1:]
train_melt=train_targets.merge(train_features,how="left",on="sig_id")
 del train_features,train_targets
train_melt.set_index("sig_id",inplace=True)
 train_melt["cp_type"]= train_melt["cp_type"].astype('category')
 train_melt["cp_dose"]= train_melt["cp_dose"].astype('category')
 return train_melt , targets.to_list()
def test_short_form_loader(feature_file):

 train_features = pd.read_csv(feature_file)
 train_features,_= reduce_mem_usage(train_features)
 train_melt = train_features.copy()
del train_features
train_melt.set_index("sig_id",inplace=True)
 train_melt["cp_type"]= train_melt["cp_type"].astype('category')
 train_melt["cp_dose"]= train_melt["cp_dose"].astype('category')
 return train_melt
def get_feature_label(data):
data_after = data[(data['price']<400)&(data['price']>1)]
train_features = data_after.drop(['price'],axis=1)
 train_labels = data_after.price
return train_features,train_labels<define_variables>
input_directory = '.. /input/lish-moa/'
train_features,train_labels=get_feature_label(train_data)
 train_features=train_features
train_labels=train_labels
test_features=test_data<feature_engineering>
train,target_cols = train_short_form_loader(input_directory +'train_features.csv',input_directory+'train_targets_scored.csv')
 test = test_short_form_loader(input_directory +"test_features.csv")
train_labels = np.log(train_labels )<feature_engineering>
GENES = [col for col in train.columns if col.startswith('g-')]
CELLS = [col for col in train.columns if col.startswith('c-')]
def category(data):
cat = data.category_name.str.split('/', expand = True)
 data["main_cat"] = cat[0]
data["subcat1"] = cat[1]
data["subcat2"] = cat[2]
try:
data["subcat3"] = cat[3]
except:
data["subcat3"] = np.nan
try:
data["subcat4"] = cat[4]
except:
data["subcat4"] = np.nan
def missing_data(data, _value = 'None'):
for col in data.columns:
data[col].fillna(_value,inplace=True )<categorify>
from sklearn.decomposition import PCA
from sklearn.preprocessing import QuantileTransformer
le = preprocessing.LabelEncoder()
def cat_to_num(train,test):
suf="_le"
for col in ['brand_name','main_cat','subcat1','subcat2','subcat3','subcat4']:
train[col+suf] = le.fit_transform(train[col])
 dic = dict(zip(le.classes_, le.transform(le.classes_)))
 test[col+suf] = test[col].map(dic ).fillna(0 ).astype(int)
 print("{} is transformed to {}".format(col,col+suf)) 
<categorify>
from sklearn.compose import make_column_transformer,ColumnTransformer
<concatenate>
def seed_everything(seed=42):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )
cat_to_num(train_features,test_features )<feature_engineering>
class CatIntMapper(BaseEstimator, TransformerMixin):
def __init__(self ,col,dicti):
self.col = col
self.dicti = dicti
def fit(self, X, y = None):
return self
def fit_transform(self, X, y = None):
assert X[self.col].isin(self.dicti.keys() ).all()
return pd.concat([X.drop(self.col,axis=1),X[self.col].map(self.dicti ).astype(int ).rename(self.col)],axis=1)
 def transform(self, X):
assert X[self.col].isin(self.dicti.keys() ).all()
return pd.concat([X.drop(self.col,axis=1),X[self.col].map(self.dicti ).astype(int ).rename(self.col)],axis=1)
 class NamedOutTWrapper(BaseEstimator, TransformerMixin):
def __init__(self,transformer,columns,inplace=False,prefix='_'):
self.transformer = transformer
self.cols = columns
self.inplace = inplace
self.prefix = prefix
self.transformer_name = self._get_transformer_name()
def fit(self, X, y = None):
self.transformer = self.transformer.fit(X[self.cols] , y)
 return self
def fit_transform(self, X, y = None):
transformed_columns = self.transformer.fit_transform(X[self.cols] , y)
 out=pd.DataFrame(index=X.index)
 if self.inplace:
out = X[self.cols]
out[self.cols] = transformed_columns
return pd.concat([X.drop(self.cols,axis=1),out],axis=1)
 else:
for i,values in enumerate(transformed_columns.transpose()):
out[ self.transformer_name + self.prefix + str(i)] = values
return pd.concat([X,out],axis=1)
 def transform(self, X):
transformed_columns = self.transformer.transform(X[self.cols])
 out=pd.DataFrame(index=X.index)
 if self.inplace:
out = X[self.cols]
out[self.cols] = transformed_columns
return pd.concat([X.drop(self.cols,axis=1),out],axis=1)
 else:
for i,values in enumerate(transformed_columns.transpose()):
out[ self.transformer_name + self.prefix + str(i)] = values
return pd.concat([X,out],axis=1)
 def _get_transformer_name(self):
return str(self.transformer.__class__ ).split('.')[-1][0:-2]
class IdentityTransformer:

 def __init__(self):
pass
def fit(self, X, y = None):
return self
def fit_transform(self, X, y = None):
return X
def transform(self, X):
return X
class SuppressControls(BaseEstimator, TransformerMixin):
def __init__(self):
pass
def fit(self, X, y = None):
return self
def fit_transform(self, X, y = None):
return X.loc[X['cp_type']!='ctl_vehicle'].drop('cp_type', axis=1)
 def transform(self, X):
return X.loc[X['cp_type']!='ctl_vehicle'].drop('cp_type', axis=1)
train_features['Length_of_item_description']=train_features['item_description'].apply(len)
 test_features['Length_of_item_description']=test_features['item_description'].apply(len )<categorify>
def multifold_indexer(train,target_columns,n_splits=10,random_state=12347,**kwargs):
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=n_splits,random_state=random_state,**kwargs)
 folds[ 'kfold']=0
for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=train[target_columns])) :
folds.iloc[v_idx,-1] = int(f)
 folds['kfold'] = folds['kfold'].astype(int)
 return folds
def numeric_to_features(data):
numeric_features = list(data.apply(lambda x:(x['shipping'],x['item_condition_id'],x['main_cat_le'],\
x['subcat1_le'],x['subcat2_le'],x['subcat3_le'],\
x['subcat4_le'],x['Length_of_item_description'],\
x['brand_name_le']), axis=1))
return numeric_features 
<categorify>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
train_numeric_features = numeric_to_features(train_features)
 test_numeric_features = numeric_to_features(test_features)
<feature_engineering>
class DAE_Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size=1100,hidden_size2=1300):
super(DAE_Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size2))
self.batch_norm3 = nn.BatchNorm1d(hidden_size2)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size2, hidden_size))
self.dense4 = nn.utils.weight_norm(nn.Linear(hidden_size, num_features))
def forward(self, x,mode='DAE'):
x1 = F.relu(self.dense1(x))
x2 = self.batch_norm2(x1)
 x2 = F.relu(self.dense2(x2))
x3 = self.batch_norm3(x2)
 x3 = F.relu(self.dense3(x3))
out = self.dense4(x3)
 if mode == 'DAE':
return out
else:
return x1,x2,x3
tfidf_d = TfidfVectorizer(sublinear_tf=True,ngram_range=(1,3),min_df=0, stop_words = 'english',max_features = 500)
 Description_matrix_train= tfidf_d.fit_transform(train_features['item_description'])
 Description_matrix_test= tfidf_d.transform(test_features['item_description'])
 tfidf_n = CountVectorizer(min_df = 10)
 name_features_train = tfidf_n.fit_transform(train_features['name'])
 name_features_test = tfidf_n.transform(test_features['name'])
 train_text_features = sparse.hstack([ name_features_train, Description_matrix_train], format='csr')
 test_text_features = sparse.hstack([ name_features_test, Description_matrix_test], format='csr')
<concatenate>
from sklearn.decomposition import PCA
from sklearn.preprocessing import QuantileTransformer
from sklearn.pipeline import make_pipeline,make_union
train_final_features = sparse.hstack(( train_numeric_features,train_text_features)).tocsr()
test_final_features = sparse.hstack(( test_numeric_features,test_text_features)).tocsr() 
<choose_model_class>
map_controls = CatIntMapper('cp_type',{'ctl_vehicle': 0, 'trt_cp': 1})
 map_dose = CatIntMapper('cp_dose',{'D1': 1, 'D2': 0})
 map_time = CatIntMapper('cp_time',{24: 0, 48: 1, 72: 2})
xgb = xgboost.XGBRegressor(n_estimators=800, learning_rate=0.25, gamma=0,booster='gbtree',n_jobs=4,subsample=1,colsample_bytree=1,min_child_weight=1, max_depth=15,seed=1505)
<predict_on_test>
train = pd.read_csv(f'{input_directory}/train_features.csv' )
xgb.fit(train_final_features,train_labels)
 predictions = xgb.predict(test_final_features )<save_to_csv>
GENES = [col for col in train.columns if col.startswith('g-')]
CELLS = [col for col in train.columns if col.startswith('c-')]
results = np.exp(predictions)
 results[results<0]=0
outfile_name = 'submit_xgboost_regression.csv'
prediction = pd.DataFrame(np.array(results), columns = ['price'])
 prediction.index.name = 'test_id'
prediction.to_csv(outfile_name, encoding='utf-8' )<set_options>
Rankg_g_tansform = NamedOutTWrapper(QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal"),columns= GENES+CELLS,inplace=True )
%matplotlib inline 
<load_from_csv>
PCA_g_tansform = NamedOutTWrapper(PCA(20),columns= GENES,prefix ='_g' )
Start_time = time.time()
train_data = pd.read_csv('.. /input/train.tsv', sep='\t')
 test_data = pd.read_csv('.. /input/test.tsv', sep='\t' )<feature_engineering>
PCA_c_tansform = NamedOutTWrapper(PCA(20),columns= CELLS,prefix ='_c' )
train_data['price'] = np.log1p(train_data['price'])
 train_data['shipping'] = np.log1p(train_data['shipping'])
 test_data['shipping'] = np.log1p(test_data['shipping'] )<prepare_x_and_y>
class ColumnDropper(BaseEstimator, TransformerMixin):
def __init__(self, cols):
self.cols=cols
def fit(self, X, y = None):
return self
def transform(self, X, y = None):
return X.drop(self.cols,axis=1)
y = train_data['price']<feature_engineering>
CatDropper =ColumnDropper(cols=['cp_type','cp_time','cp_dose'] )
def if_catname(row):

 if row == row:
return 0
else:
return 1
train_data['Category_missing'] = train_data.category_name.apply(lambda row : if_catname(row))
train_data['Brand_missing'] = train_data.brand_name.apply(lambda row : if_catname(row))
train_data['Item_missing'] = train_data.item_description.apply(lambda row : if_catname(row))
test_data['Category_missing'] = test_data.category_name.apply(lambda row : if_catname(row))
test_data['Brand_missing'] = test_data.brand_name.apply(lambda row : if_catname(row))
test_data['Item_missing'] = test_data.item_description.apply(lambda row : if_catname(row)) 
<categorify>
transformers_list=[map_controls,map_dose,map_time,Rankg_g_tansform,CatDropper]
train_data["category_name"].fillna("None/None/None", inplace=True)
 test_data["category_name"].fillna("None/None/None", inplace=True)
 train_data["brand_name"].fillna("None", inplace=True)
 test_data["brand_name"].fillna("None", inplace=True)
 train_data["item_description"].fillna("None", inplace=True)
 test_data["item_description"].fillna("None", inplace=True)
<define_variables>
exp_name = 'test_DAE_0.3_all_together'
ID_train = train_data['train_id']
ID_test = test_data['test_id']<drop_column>
def run_inference(X_train,y_train,X_valid,y_valid,X_test,fold, seed,inference_only=False,**kwargs):
seed_everything(seed)
 if not inference_only:
train_dataset = MoADataset(X_train, y_train)
 valid_dataset = MoADataset(X_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 testdataset = TestDataset(X_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = DAE_Model(
 num_features= X_train.shape[1] ,
num_targets= X_train.shape[1],
**kwargs 
)
 model.load_state_dict(torch.load(f"{BOOK1}FOLD{fold}_{exp_name}.pth",map_location=torch.device('cpu')))
 print('
model.to(DEVICE)
 if not inference_only:
oof = inference_infer_features_fn(model, validloader, DEVICE)
 else:
oof= 0
predictions = infer_features_fn(model, testloader, DEVICE)
 predictions = predictions
return oof, predictions
train_data.drop("train_id", axis = 1, inplace = True)
 test_data.drop("test_id", axis = 1, inplace = True)
 train_data.drop("price", axis = 1, inplace = True)
<concatenate>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 1000
BATCH_SIZE = 640
LEARNING_RATE = 2e-3
WEIGHT_DECAY = 1e-8
NFOLDS = 10
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
GAMMA=0.5
FACTOR=0.75
hidden_size=1100
hidden_size2=1300
PATIENCE=10
THRESHOLD = 5e-3
Combined_data = pd.concat([train_data,test_data] ).reset_index(drop=True )<filter>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
if not scheduler.__class__ == torch.optim.lr_scheduler.ReduceLROnPlateau:
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, scheduler, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 if scheduler.__class__ == torch.optim.lr_scheduler.ReduceLROnPlateau:
scheduler.step(final_loss)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
def infer_features_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs,mode='get_features')
 preds.append(torch.cat(outputs,axis=1 ).detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
Combined_data_numerical = Combined_data[numerical_features] 
<feature_engineering>
SEED = [0]
train,target_cols = train_short_form_loader('.. /input/lish-moa/train_features.csv','.. /input/lish-moa/train_targets_scored.csv')
 test = test_short_form_loader(".. /input/lish-moa/test_features.csv")
 train = pd.concat([train,test])
 train[target_cols]= train[target_cols].fillna(0)
 test = train.copy()
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
train = multifold_indexer(train,target_cols,n_splits=NFOLDS)
 for fold in range(NFOLDS):
train_df = train[train['kfold'] != fold]
valid_df = train[train['kfold'] == fold]
feature_cols = [col for col in train_df.columns if not(col in target_cols+['kfold'])]
pipeline_val = make_pipeline(*transformers_list)
 X_train, y_train = train_df[feature_cols], train_df[target_cols]
X_valid, y_valid = valid_df[feature_cols], valid_df[target_cols].values
pipeline_val = load(BOOK2+'pipeline_val2_fold%i.joblib'%fold)
 print('LOADED PIPELINE_VAL2_FOLD%i'%fold)
 print(pipeline_val)
 X_train = pipeline_val.transform(X_train)
 X_train = X_train.values
X_valid = pipeline_val.transform(X_valid)
 valid_index = X_valid.index
X_valid = X_valid.values
y_train = y_train.values
X_test = test[feature_cols]
X_test = pipeline_val.transform(X_test ).values
pred_ = run_inference(X_train,y_train,X_valid,y_valid,X_test,fold, seed,inference_only=True)
 print('
transformed_features = pd.DataFrame(pred_[1],index=test.index)
 transformed_features.columns = [str(i)for i in range(len(transformed_features.columns)) ]
transformed_features.reset_index().to_feather('./features_0.2_altogether_fold%i.fth'%fold)
 if fold>=(CTT-1): break
count = CountVectorizer(min_df=10)
 X_name = count.fit_transform(Combined_data['name'])
 X_name.shape<categorify>
from sklearn.base import BaseEstimator, TransformerMixin
vector_brand = LabelBinarizer(sparse_output=True)
 X_brand = vector_brand.fit_transform(Combined_data['brand_name'])
 X_brand.shape<prepare_x_and_y>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
vector_L0 = LabelBinarizer(sparse_output=True)
 X_L0 = vector_brand.fit_transform(Combined_data['category_name'])
 X_L0.shape<prepare_x_and_y>
def seed_everything(seed=42):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
tfidf_vec = TfidfVectorizer(stop_words='english', ngram_range=(1,2), max_features=50000)
 X_description = tfidf_vec.fit_transform(Combined_data['item_description'])
 X_description.shape<concatenate>
class DaeAdder(BaseEstimator, TransformerMixin):
def __init__(self,filename):
self.filename=filename
def fit(self,X,y=None):
return self
def fit_transform(self,X,y=None):
Dae_features = pd.read_feather(self.filename ).set_index('sig_id')
 return X.merge(Dae_features,how='left', on='sig_id')
 def transform(self,X):
Dae_features = pd.read_feather(self.filename ).set_index('sig_id')
 return X.merge(Dae_features,how='left', on='sig_id')
X = sparse.hstack(( X_name, X_brand, X_L0, num_features, X_description)).tocsr()
X.shape<compute_train_metric>
class SupressControls(BaseEstimator, TransformerMixin):
def __init__(self):
pass
def fit(self, X, y = None):
return self
def fit_transform(self, X, y = None):
X = X[X['cp_type']!=0]
X = X.drop('cp_type', axis=1)
 return X
def transform(self, X):
X = X[X['cp_type']!=0]
X = X.drop('cp_type', axis=1)
 return X
class CatIntMapper(BaseEstimator, TransformerMixin):
def __init__(self ,col,dicti):
self.col = col
self.dicti = dicti
def fit(self, X, y = None):
return self
def fit_transform(self, X, y = None):
assert X[self.col].isin(self.dicti.keys() ).all()
return pd.concat([X.drop(self.col,axis=1),X[self.col].map(self.dicti ).astype(int ).rename(self.col)],axis=1)
 def transform(self, X):
assert X[self.col].isin(self.dicti.keys() ).all()
return pd.concat([X.drop(self.col,axis=1),X[self.col].map(self.dicti ).astype(int ).rename(self.col)],axis=1)
 class NamedOutTWrapper(BaseEstimator, TransformerMixin):
def __init__(self,transformer,columns,inplace=False,prefix='_'):
self.transformer = transformer
self.cols = columns
self.inplace = inplace
self.prefix = prefix
self.transformer_name = self._get_transformer_name()
def fit(self, X, y = None):
self.transformer = self.transformer.fit(X[self.cols] , y)
 return self
def fit_transform(self, X, y = None):
transformed_columns = self.transformer.fit_transform(X[self.cols] , y)
 out=pd.DataFrame(index=X.index)
 if self.inplace:
out = X[self.cols].copy()
out[self.cols] = transformed_columns
return pd.concat([X.drop(self.cols,axis=1),out],axis=1)
 else:
for i,values in enumerate(transformed_columns.transpose()):
out[ self.transformer_name + self.prefix + str(i)] = values
return pd.concat([X,out],axis=1)
 def transform(self, X):
transformed_columns = self.transformer.transform(X[self.cols])
 out=pd.DataFrame(index=X.index)
 if self.inplace:
out = X[self.cols].copy()
out[self.cols] = transformed_columns
return pd.concat([X.drop(self.cols,axis=1),out],axis=1)
 else:
for i,values in enumerate(transformed_columns.transpose()):
out[ self.transformer_name + self.prefix + str(i)] = values
return pd.concat([X,out],axis=1)
 def _get_transformer_name(self):
return str(self.transformer.__class__ ).split('.')[-1][0:-2]
class IdentityTransformer:

 def __init__(self):
pass
def fit(self, X, y = None):
return self
def fit_transform(self, X, y = None):
return X
def transform(self, X):
return X
class SuppressControls(BaseEstimator, TransformerMixin):
def __init__(self):
pass
def fit(self, X, y = None):
return self
def fit_transform(self, X, y = None):
return X.loc[X['cp_type']=='trt_cp'].drop('cp_type', axis=1)
 def transform(self, X):
return X.loc[X['cp_type']=='trt_cp'].drop('cp_type', axis=1)
 class ColumnDropper(BaseEstimator, TransformerMixin):
def __init__(self, cols):
self.cols=cols
def fit(self, X, y = None):
return self
def transform(self, X, y = None):
return X.drop(self.cols,axis=1)
 class VarianceFilter(BaseEstimator, TransformerMixin):
def __init__(self,threshold):
self.threshold = threshold
def fit(self,X,y=None):
mask = X.var() <= self.threshold
self.drop_cols = set([ col for val,col in zip(mask,X.columns)if val])
 self.drop_cols.discard('cp_type')
 return self
def transform(self,X):
return X.drop(self.drop_cols,axis=1)
 def apply_pipe_together(pipeline,train,test,ct=None):
data = pd.concat([train,test])
 pipeline = load(BOOK2+'pipe_together_ct%i.joblib'%ct)
 print('LOAD PIPE_TOGETHER_CT%i'%ct)
 print(pipeline)
 data = pipeline.transform(data)
 train = data.loc[data.index.intersection(train.index)]
test = data.loc[data.index.intersection(test.index)]
return pipeline,train,test
def evaluate_model(X, y, algorithm):
X_train, X_test, y_train, y_test = train_test_split(train, y, test_size = 0.20, random_state = 1)
 print(algorithm)
 print()
print('Train - Mean Squared Error')
 print(( np.sqrt(-cross_val_score(algorithm, X_train, y_train, scoring="neg_mean_squared_error", cv = 2)).mean()))
print()
print('Test - Mean Squared Error')
 print(( np.sqrt(-cross_val_score(algorithm, X_test, y_test, scoring="neg_mean_squared_error", cv = 2)).mean()))
print() 
<train_model>
map_controls = CatIntMapper('cp_type',{'ctl_vehicle': 0, 'trt_cp': 1})
 map_dose = CatIntMapper('cp_dose',{'D1': 1, 'D2': 0})
 map_time = CatIntMapper('cp_time',{24: 0, 48: 1, 72: 2})
ridge = Ridge(solver='sag', fit_intercept=True, alpha=0.02, max_iter=200, normalize=False, tol=0.01)
 ridge.fit(train, y )<predict_on_test>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
labels_ridge = np.expm1(ridge.predict(test))<save_to_csv>
def multifold_indexer(train,target_columns,n_splits=10,random_state=12347,**kwargs):
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=n_splits,random_state=random_state,**kwargs)
 folds[ 'kfold']=0
for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=train[target_columns])) :
folds.iloc[v_idx,-1] = int(f)
 folds['kfold'] = folds['kfold'].astype(int)
 return folds
pd.DataFrame({'test_id': ID_test , 'price': labels_ridge} ).to_csv('MercariPredictions.csv', index =False )<define_variables>
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size1=388,hidden_size2=512,drop_rate1=0.0,drop_rate2=0.3,drop_rate3=0.3):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(drop_rate1)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size1))
self.batch_norm2 = nn.BatchNorm1d(hidden_size1)
 self.dropout2 = nn.Dropout(drop_rate2)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size1, hidden_size2))
self.batch_norm3 = nn.BatchNorm1d(hidden_size2)
 self.dropout3 = nn.Dropout(drop_rate3)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size2, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
end_time = time.time()<compute_test_metric>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
if not scheduler.__class__ == torch.optim.lr_scheduler.ReduceLROnPlateau:
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, scheduler, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 if scheduler.__class__ == torch.optim.lr_scheduler.ReduceLROnPlateau:
scheduler.step(final_loss)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
( end_time - Start_time )<set_options>
exp_name = 'DAE'
pd.set_option('display.float_format', lambda x: '%.3f' % x )<set_options>
VALIDATE = True
pyximport.install()
os.environ['PYTHONHASHSEED'] = '10000'
np.random.seed(10001)
 random.seed(10002)
 session_conf = tf.ConfigProto(intra_op_parallelism_threads=5, inter_op_parallelism_threads=1)
 tf.set_random_seed(10003)
 backend.set_session(tf.Session(graph=tf.get_default_graph() , config=session_conf))<compute_test_metric>
def run_training(X_train,y_train,X_valid,y_valid,X_test,fold, seed,verbose=False,ct=None,**kwargs):
seed_everything(seed)
 train_dataset = MoADataset(X_train, y_train)
 valid_dataset = MoADataset(X_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features= X_train.shape[1] ,
num_targets= y_train.shape[1],hidden_size1=hidden_size1,hidden_size2=hidden_size2,
**kwargs 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer,patience=3)
 loss_val = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
best_loss = np.inf
if 0:
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 if verbose &(epoch%10==0):
print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model,scheduler, loss_val, validloader, DEVICE)
 if verbose &(epoch%10==0):
print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof = valid_preds
elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
testdataset = TestDataset(X_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features= X_train.shape[1] ,
num_targets= y_train.shape[1],
hidden_size1=hidden_size1,
hidden_size2=hidden_size2,
**kwargs 
)
 model.load_state_dict(torch.load(f"{BOOK2}{exp_name}_SEED{seed}_FOLD{fold}_CT{ct}.pth"))
model.to(DEVICE)
 print('LOAD MODEL_FOLD%i_CT%i'%(fold,ct))
if VALIDATE:
valid_loss, oof = valid_fn(model,scheduler, loss_val, validloader, DEVICE)
 else:
oof = np.zeros(( len(X_valid),y_train.shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
def rmsle_cust(y_true, y_pred):
first_log = K.log(K.clip(y_pred, K.epsilon() , None)+ 1.)
second_log = K.log(K.clip(y_true, K.epsilon() , None)+ 1.)
return K.sqrt(K.mean(K.square(first_log - second_log), axis=-1))
def rmsle_log(Y, Y_pred):
assert Y.shape == Y_pred.shape
return np.sqrt(np.mean(np.square(Y_pred - Y)))
 def rmsle(h, y):
return np.sqrt(np.square(np.log(h + 1)- np.log(y + 1)).mean())
 def load_data(path, sep=','):
train = pd.read_csv(path, sep='\t')
 return train
def preprocess_rnn(train_data, test_data):
print('Dropping extra columns and splitting data into x and y')
 y_train = train_data['price']
x_train = train.drop(['train_id', 'price'], axis=1)
 x_test = test.drop(['test_id'], axis=1)
 print('Filling in missing values')
 x_train.item_description.fillna(value="missing", inplace=True)
 x_train.brand_name.fillna(value="missing", inplace=True)
 x_train.category_name.fillna(value='missing', inplace=True)
 x_test.item_description.fillna(value="missing", inplace=True)
 x_test.brand_name.fillna(value="missing", inplace=True)
 x_test.category_name.fillna(value='missing', inplace=True)
 print('Transforming categorical values to lowercase')
 train_name = x_train['name'].str.lower().values
train_brand_name = x_train['brand_name'].str.lower().values
train_item_description = x_train['item_description'].str.lower().values
train_category_name = x_train['category_name'].str.lower().values
train_item_condition_id = x_train['item_condition_id'].values
train_shipping = x_train['shipping'].values
test_name = x_test['name'].str.lower().values
test_brand_name = x_test['brand_name'].str.lower().values
test_item_description = x_test['item_description'].str.lower().values
test_category_name = x_test['category_name'].str.lower().values
test_item_condition_id = x_test['item_condition_id'].values
test_shipping = x_test['shipping'].values
print('Fitting the Tokenizer on the categorical data')
 tokenizer = Tokenizer(num_words=1000)
 tokenizer.fit_on_texts(np.concatenate([train_name, train_brand_name, train_category_name,
test_name, test_brand_name, test_category_name]))
print('Transforming categorical data using the tokenizr')
 train_name = tokenizer.texts_to_sequences(train_name)
 train_brand_name = tokenizer.texts_to_sequences(train_brand_name)
 train_item_description = tokenizer.texts_to_sequences(train_item_description)
 train_category_name = tokenizer.texts_to_sequences(train_category_name)
 test_name = tokenizer.texts_to_sequences(test_name)
 test_brand_name = tokenizer.texts_to_sequences(test_brand_name)
 test_item_description = tokenizer.texts_to_sequences(test_item_description)
 test_category_name = tokenizer.texts_to_sequences(test_category_name)
 print('Split data into train and val')
 name_train, name_val, brand_name_train, brand_name_val, item_description_train, \
item_description_val, category_name_train, category_name_val, item_condition_id_train, \
item_condition_id_val, shipping_train, shipping_val, y_train, y_val = \
train_test_split(train_name, train_brand_name, train_item_description, train_category_name, \
train_item_condition_id, train_shipping, y_train, test_size=0.01)
 print('Padding sequences to set length')
 x_train = {
'name': pad_sequences(name_train, 5),
'item_description': pad_sequences(item_description_train, 25),
'brand_name': pad_sequences(brand_name_train, 2),
'category_name': pad_sequences(category_name_train, 2),
'item_condition_id': np.array(item_condition_id_train),
'shipping': np.array(shipping_train)
 }
x_val = {
'name': pad_sequences(name_val, 5),
'item_description': pad_sequences(item_description_val, 25),
'brand_name': pad_sequences(brand_name_val, 2),
'category_name': pad_sequences(category_name_val, 2),
'item_condition_id': np.array(item_condition_id_val),
'shipping': np.array(shipping_val)
 }
x_test = {
'name': pad_sequences(test_name, 5),
'item_description': pad_sequences(test_item_description, 25),
'brand_name': pad_sequences(test_brand_name, 2),
'category_name': pad_sequences(test_category_name, 2),
'item_condition_id': np.array(test_item_condition_id),
'shipping': np.array(test_shipping)
 }
return x_train, y_train, x_val, y_val, x_test
def preprocess_ridge(train_data, test_data,):
print('Dropping extra columns and splitting data into x and y')
 y_train = train_data['price']
x_train = train.drop(['train_id', 'price'], axis=1)
 x_test = test.drop(['test_id'], axis=1)
 print('Filling in missing values')
 x_train.item_description.fillna(value="missing", inplace=True)
 x_train.brand_name.fillna(value="missing", inplace=True)
 x_train.category_name.fillna(value='missing', inplace=True)
 x_test.item_description.fillna(value="missing", inplace=True)
 x_test.brand_name.fillna(value="missing", inplace=True)
 x_test.category_name.fillna(value='missing', inplace=True)
 print('Use label encoder on categorical data')
 le = LabelEncoder()
le.fit(np.concatenate([x_train['category_name'], x_test['category_name']]))
x_train['category_name'] = le.transform(x_train['category_name'])
 x_test['category_name'] = le.transform(x_test['category_name'])
 le = LabelEncoder()
le.fit(np.concatenate([x_train['brand_name'], x_test['brand_name']]))
x_train['brand_name'] = le.transform(x_train['brand_name'] ).astype('str')
 x_test['brand_name'] = le.transform(x_test['brand_name'] ).astype('str')
 print('Split data into train and val')
 x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=0.01)
 print(len(y_train), len(x_train), len(x_val), len(x_test))
return x_train, y_train, x_val, y_val, x_test
def predict_rnn(model, data):
preds = model.predict(data, verbose=True)
 preds = np.abs(preds)
 return preds
def predict_ridge(model, data):
preds = model.predict(data)
 preds = np.abs(preds)
 return preds
def submit(preds):
index = pd.read_csv(f'.. /input/test.tsv', sep='\t')['test_id'].values
submission = pd.DataFrame(index, columns=['test_id'], index=None)
 submission['price'] = preds
submission.to_csv('submission.csv',index=False )<prepare_x_and_y>
DEVICE = torch.device('cuda:0')
 EPOCHS = 36
BATCH_SIZE = 1024
LEARNING_RATE = 6e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
hidden_size1=2048
hidden_size2=2048
%%time
train = load_data('.. /input/train.tsv', '\t')
 test = load_data('.. /input/test.tsv', '\t')
 x_train_rnn, y_train_rnn, x_val_rnn, y_val_rnn, x_test_rnn = preprocess_rnn(train, test)
 x_train_ridge, y_train_ridge, x_val_ridge, y_val_ridge, x_test_ridge = preprocess_ridge(train, test )<feature_engineering>
def run_k_fold(folds,target_cols,test,transformers_list,NFOLDS, seed,verbose=False,ct=None, **kwargs):
train = folds
test_ = test
oof = train[target_cols].copy()
oof = oof*0
predictions = pd.DataFrame(0,columns=target_cols,index=test.index)
 for fold in range(NFOLDS):
train_df = train[train['kfold'] != fold]
valid_df = train[train['kfold'] == fold]
feature_cols = [col for col in train_df.columns if not(col in target_cols+['kfold'])]
pipeline_val = make_pipeline(*transformers_list)
 X_train, y_train = train_df[feature_cols], train_df[target_cols]
X_valid, y_valid = valid_df[feature_cols], valid_df[target_cols].values
X_train = pipeline_val.fit_transform(X_train,y_train)
 feature_cols = [col for col in X_train.columns if not(col in target_cols+['kfold'])]
X_train = X_train.values
X_valid = pipeline_val.transform(X_valid)
 valid_index = X_valid.index
X_valid = X_valid.values
y_train = y_train.values
X_test = pipeline_val.transform(test_)
 test_index = X_test.index
X_test = X_test[feature_cols].values
oof_, pred_ = run_training(X_train,y_train,X_valid,y_valid,X_test,fold, seed,verbose,ct=ct,**kwargs)
 oof.loc[valid_index] = oof_
predictions.loc[test_index] += pred_ / NFOLDS
return oof, predictions
y_train_rnn = np.log1p(y_train_rnn)
 y_val_rnn = np.log1p(y_val_rnn )<categorify>
params={}
name_input = Input(shape=[x_train_rnn["name"].shape[1]], name='name')
 item_description_input = Input(shape=[25], name='item_description')
 brand_name_input = Input(shape=[2], name='brand_name')
 category_name_input = Input(shape=[2], name='category_name')
 item_condition_id_input = Input(shape=[1], name='item_condition_id')
 shipping_input = Input(shape=[1], name='shipping')
 name_embedding = Embedding(1000, 20 )(name_input)
 item_description_embedding = Embedding(1000, 75 )(item_description_input)
 brand_name_embedding = Embedding(1000, 10 )(brand_name_input)
 category_name_embedding = Embedding(1000, 10 )(category_name_input)
 name_rnn = GRU(8 )(name_embedding)
 item_description_rnn = GRU(16 )(item_description_embedding)
 concatenate_1 = concatenate([ 
(name_rnn), 
(item_description_rnn),
Flatten()(brand_name_embedding),
Flatten()(category_name_embedding), 
(item_condition_id_input), 
(shipping_input),
])
 dense_1 = Dense(256, activation='relu' )(concatenate_1)
 dropout_1 = Dropout(0.2 )(dense_1)
 dense_2 = Dense(128, activation='relu' )(dropout_1)
 dropout_2 = Dropout(0.2 )(dense_2)
 dense_3 = Dense(64, activation='relu' )(dropout_2)
 dropout_3 = Dropout(0.2 )(dense_3)
 output = Dense(1, activation="linear" )(dropout_3)
 rnn_model = Model([name_input, brand_name_input, item_description_input, category_name_input,
item_condition_id_input, shipping_input], output)
 rnn_model.compile(loss="mse", optimizer=keras.optimizers.Adam() , metrics=['mae', rmsle_cust])
 rnn_model.summary()<train_model>
k = 0
SEED = [0]
Dae0_2 = DaeAdder(filename='features_0.2_altogether_fold%i.fth'%k)
 transformers_list=[map_dose,map_time,Dae0_2,SuppressControls() ]
train,target_cols = train_short_form_loader('.. /input/lish-moa/train_features.csv','.. /input/lish-moa/train_targets_scored.csv')
 test = test_short_form_loader(".. /input/lish-moa/test_features.csv")
 pipeline_test = make_pipeline(*transformers_list)
 pipeline_test,train , test = apply_pipe_together(pipeline_test,train,test,ct=k)
 oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for k in range(CTT):
if k!=0:
Dae0_2 = DaeAdder(filename='features_0.2_altogether_fold%i.fth'%k)
 transformers_list=[map_dose,map_time,Dae0_2,SuppressControls() ]
train,target_cols = train_short_form_loader('.. /input/lish-moa/train_features.csv','.. /input/lish-moa/train_targets_scored.csv')
 test = test_short_form_loader(".. /input/lish-moa/test_features.csv")
 pipeline_test = make_pipeline(*transformers_list)
 pipeline_test,train , test = apply_pipe_together(pipeline_test,train,test,ct=k)
 transformers_list=[IdentityTransformer() ]
for seed in SEED:
folds = multifold_indexer(train,target_cols,n_splits=NFOLDS)
 oof_, predictions_ = run_k_fold(folds,target_cols,test,transformers_list,NFOLDS, seed,verbose=True,ct=k,**params)
 oof += oof_ / len(SEED)/ CTT
predictions += predictions_ / len(SEED)/ CTT
print()
test[target_cols] = predictions
%%time
rnn_model.fit(x_train_rnn, y_train_rnn, batch_size=64, validation_data=(x_val_rnn, y_val_rnn), epochs=1 )<predict_on_test>
y_true = train[target_cols].values
y_pred = oof
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred.iloc[:, i],labels=[0,1])
 score +=(score_ / len(target_cols))
print('
print("CV log_loss: ", score)
rnn_val_preds = rnn_model.predict(x_val_rnn, verbose=1)
 rnn_train_preds = rnn_model.predict(x_train_rnn, verbose=1 )<compute_test_metric>
train,target_cols = train_short_form_loader('.. /input/lish-moa/train_features.csv','.. /input/lish-moa/train_targets_scored.csv')
 y_true = train[target_cols].values
y_pred = train[target_cols].copy()
y_pred[target_cols] = 0
y_pred.loc[oof.index] = oof
y_pred.loc[train.cp_type=='ctl_vehicle'] = 0
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred.iloc[:, i],labels=[0,1])
 score +=(score_ / len(target_cols))
print('
print("CV log_loss: ", score)
rmsle(np.expm1(rnn_train_preds), np.expm1(y_train_rnn.values.reshape(-1, 1)) )<compute_test_metric>
sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission.set_index('sig_id',inplace=True)
 test_features.set_index('sig_id',inplace=True)
 test_features = test_features.loc[sample_submission.index]
sub = sample_submission.drop(columns=target_cols ).merge(test[target_cols], on='sig_id', how='left' ).fillna(0)
 sub.loc[test_features['cp_type']=='ctl_vehicle', target_cols] =0
sub.to_csv('./submission.csv', index=True)
 sub.head()
<compute_test_metric><EOS> 
s = sub.loc[test_features['cp_type']=='ctl_vehicle', target_cols].sum().sum()
print('Sub control rows =',s )