Kaggle competition: Mechanisms of Action (MoA) Prediction
testo =test.copy()<drop_column>
warnings.filterwarnings('ignore' )
df.drop(columns=['Name', 'Ticket', 'Cabin'], axis=1, inplace=True)
 test.drop(columns=['Name', 'Ticket', 'Cabin'], axis=1, inplace=True )<count_missing_values>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
df.isnull().sum()<data_type_conversions>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
df['Age'].fillna(df['Age'].median() , inplace=True)
 df['Embarked'].fillna(df['Embarked'].mode() [0], inplace=True)
<count_missing_values>
def seed_everything(seed=321):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=321 )
df.isnull().sum()<filter>
n_comp = 29
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=321 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 n_comp = 5
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=321 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
df[df.Fare.isnull() ] 
<count_missing_values>
var_thresh = VarianceThreshold(threshold=0.4)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features
df.isnull().sum()<count_missing_values>
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1)
test.isnull().sum()<count_missing_values>
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
test.isnull().sum()<count_duplicates>
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=10)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int )
print('Duplicated data =',df.duplicated().sum() )<sort_values>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
df[['Pclass','Survived']].groupby(['Pclass'],as_index = False ).mean().sort_values(by = 'Survived', ascending = False )<groupby>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
df[['Sex','Survived']].groupby(['Sex'],as_index = False ).mean()<groupby>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
df[['SibSp','Survived']].groupby(['SibSp'] ).mean()<groupby>
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.activation1 = torch.nn.PReLU(num_parameters = hidden_size, init = 1.0)
 self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.3)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.activation2 = torch.nn.PReLU(num_parameters = hidden_size, init = 0.25)
 self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.25)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def recalibrate_layer(self, layer):
if(torch.isnan(layer.weight_v ).sum() > 0):
print('recalibrate layer.weight_v')
 layer.weight_v = torch.nn.Parameter(torch.where(torch.isnan(layer.weight_v), torch.zeros_like(layer.weight_v), layer.weight_v))
layer.weight_v = torch.nn.Parameter(layer.weight_v + 1e-7)
 if(torch.isnan(layer.weight ).sum() > 0):
print('recalibrate layer.weight')
 layer.weight = torch.where(torch.isnan(layer.weight), torch.zeros_like(layer.weight), layer.weight)
 layer.weight += 1e-7
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 self.recalibrate_layer(self.dense1)
 x = self.activation1(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 self.recalibrate_layer(self.dense2)
 x = self.activation2(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 self.recalibrate_layer(self.dense3)
 x = self.dense3(x)
 return x
df[['Parch','Survived']].groupby(['Parch'] ).mean()<categorify>
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
LabelEncoder = preprocessing.LabelEncoder()
df['Embarked'] = LabelEncoder.fit_transform(df['Embarked'])
 test['Embarked'] = LabelEncoder.transform(test['Embarked'])
 df['Sex'] = LabelEncoder.fit_transform(df['Sex'])
 test['Sex'] = LabelEncoder.fit_transform(test['Sex'])
 y = df['Survived']
X = df.drop(['Survived'],axis=1 )<split>
feature_cols = [c for c in process_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25 )<normalization>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 25
BATCH_SIZE = 512
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 10
EARLY_STOPPING_STEPS = 25
EARLY_STOP = True
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=2048
scaler = MinMaxScaler()
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test)
 test = scaler.fit_transform(test )<train_model>
def run_training(fold, seed):
seed_everything(seed)
 train = process_data(folds)
 test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.0005)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
def perform_model(model, X_train, y_train, X_test, y_test, class_labels, cm_normalize=True, \
print_cm=True, cm_cmap=plt.cm.Greens):
results = dict()
train_start_time = datetime.now()
print('training the model.. ')
 model.fit(X_train, y_train)
 print('Done
')
 train_end_time = datetime.now()
results['training_time'] = train_end_time - train_start_time
print('training_time(HH:MM:SS.ms)- {}
'.format(results['training_time']))
print('Predicting test data')
 test_start_time = datetime.now()
y_pred = model.predict(X_test)
 test_end_time = datetime.now()
print('Done
')
 results['testing_time'] = test_end_time - test_start_time
print('testing time(HH:MM:SS:ms)- {}
'.format(results['testing_time']))
results['predicted'] = y_pred
accuracy = accuracy_score(y_true=y_test, y_pred=y_pred)
 results['accuracy'] = accuracy
print('---------------------')
 print('| Accuracy |')
 print('---------------------')
 print('
{}
'.format(accuracy))
cm = confusion_matrix(y_test, y_pred)
 results['confusion_matrix'] = cm
if print_cm:
print('--------------------')
 print('| Confusion Matrix |')
 print('--------------------')
 print('
{}'.format(cm))
plt.figure(figsize=(4,4))
plt.grid(b=False)
 plot_confusion_matrix(cm, classes=class_labels, normalize=True, title='Normalized confusion matrix', cmap = cm_cmap)
 plt.show()
print('-------------------------')
 print('| Classifiction Report |')
 print('-------------------------')
 classification_Report = classification_report(y_test, y_pred)
 results['classification_Report'] = classification_Report
print(classification_Report)
 results['model'] = model
return results 
<find_best_params>
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
def print_grid_search_attributes(model):
print('--------------------------')
 print('| Best Estimator |')
 print('--------------------------')
 print('
\t{}
'.format(model.best_estimator_))
print('--------------------------')
 print('| Best parameters |')
 print('--------------------------')
 print('\tParameters of best estimator :
\t{}
'.format(model.best_params_))
print('---------------------------------')
 print('| No of CrossValidation sets |')
 print('--------------------------------')
 print('
\tTotal numbre of cross validation sets: {}
'.format(model.n_splits_))
print('--------------------------')
 print('| Best Score |')
 print('--------------------------')
 print('
\tAverage Cross Validate scores of best estimator :
\t{}
'.format(model.best_score_))<define_variables>
SEED = [456]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions
labels = ['0','1']<train_on_grid>
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score )
parameters = {"C":np.logspace(-3,3,7), "penalty":["l1","l2"]}
logreg=LogisticRegression()
lr_grid = GridSearchCV(logreg,param_grid=parameters, n_jobs=-1)
 lr_grid_results = perform_model(lr_grid, X_train, y_train, X_test, y_test, class_labels=labels)
 print_grid_search_attributes(lr_grid_results['model'] )<train_on_grid>
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )