Kaggle competition: Don't Overfit! II
<SOS> metric: AUC Kaggle data source: dont-overfit!-ii<define_variables>
%matplotlib inline
exp_name="original_torch_moa_5_folds_continued" 
<set_options>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv" )
def reduce_mem_usage(props):
start_mem_usg = props.memory_usage().sum() / 1024**2
print("Memory usage of properties dataframe is :",start_mem_usg," MB")
 NAlist = []
for col in props.columns:
if props[col].dtype != object:
IsInt = False
mx = props[col].max()
mn = props[col].min()
if not np.isfinite(props[col] ).all() :
NAlist.append(col)
 props[col].fillna(mn-1,inplace=True)
 asint = props[col].fillna(0 ).astype(np.int64)
 result =(props[col] - asint)
 result = result.sum()
if result > -0.01 and result < 0.01:
IsInt = True
if IsInt:
if mn >= 0:
if mx < 255:
props[col] = props[col].astype(np.uint8)
 elif mx < 65535:
props[col] = props[col].astype(np.uint16)
 elif mx < 4294967295:
props[col] = props[col].astype(np.uint32)
 else:
props[col] = props[col].astype(np.uint64)
 else:
if mn > np.iinfo(np.int8 ).min and mx < np.iinfo(np.int8 ).max:
props[col] = props[col].astype(np.int8)
 elif mn > np.iinfo(np.int16 ).min and mx < np.iinfo(np.int16 ).max:
props[col] = props[col].astype(np.int16)
 elif mn > np.iinfo(np.int32 ).min and mx < np.iinfo(np.int32 ).max:
props[col] = props[col].astype(np.int32)
 elif mn > np.iinfo(np.int64 ).min and mx < np.iinfo(np.int64 ).max:
props[col] = props[col].astype(np.int64)
 else:
props[col] = props[col].astype(np.float32)
 print("___MEMORY USAGE AFTER COMPLETION:___")
 mem_usg = props.memory_usage().sum() / 1024**2
print("Memory usage is: ",mem_usg," MB")
 print("This is ",100*mem_usg/start_mem_usg,"% of the initial size")
 return props, NAlist 
<load_from_csv>
train_df=reduce_mem_usage(train_df)
 test_df=reduce_mem_usage(test_df )
def train_short_form_loader(feature_file,target_file,extra_target_file=None):

 train_features = pd.read_csv(feature_file)
 train_targets = pd.read_csv(target_file)
 train_features,_= reduce_mem_usage(train_features)
 train_targets,_ = reduce_mem_usage(train_targets)
 if extra_target_file is not None:
extra_targets = pd.read_csv(extra_target_file)
 extra_targets,_ = reduce_mem_usage(extra_targets)
 train_targets = pd.concat([train_targets,extra_targets])
 del extra_targets
targets = train_targets.columns[1:]
train_melt=train_targets.merge(train_features,how="left",on="sig_id")
 del train_features,train_targets
train_melt.set_index("sig_id",inplace=True)
 train_melt["cp_type"]= train_melt["cp_type"].astype('category')
 train_melt["cp_dose"]= train_melt["cp_dose"].astype('category')
 return train_melt , targets
def test_short_form_loader(feature_file):

 train_features = pd.read_csv(feature_file)
 train_features,_= reduce_mem_usage(train_features)
 train_melt = train_features.copy()
del train_features
train_melt.set_index("sig_id",inplace=True)
 train_melt["cp_type"]= train_melt["cp_type"].astype('category')
 train_melt["cp_dose"]= train_melt["cp_dose"].astype('category')
 return train_melt 
<define_variables>
target = train_df["target"]
train_df = train_df.drop(["target","id"],axis=1)
 test_id = test_df["id"]
test_df = test_df.drop(["id"],axis=1 )
train,target=train_short_form_loader('.. /input/lish-moa/train_features.csv','.. /input/lish-moa/train_targets_scored.csv' )<load_from_csv>
clf = LogisticRegression(class_weight = 'balanced',
penalty='l1',
C=0.1,
solver='liblinear' ).fit(train_df, target)
<define_variables>
np.sum(clf.coef_!=0 )
<set_options>
train_short=train_df.iloc[:,clf.coef_[0]!=0]
test_short=test_df.iloc[:,clf.coef_[0]!=0]
def seed_everything(seed=42):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )<drop_column>
cmb_df= pd.concat([train_short,test_short] )
def supress_controls(df):
df = df[train['cp_type']!='ctl_vehicle']
df = df.drop('cp_type', axis=1)
 return df<categorify>
cmb_df = pd.get_dummies(cmb_df,dummy_na= False )
def map_controls(df):
df['cp_type']=df['cp_type'].map({'ctl_vehicle': 0, 'trt_cp': 1})
 df['cp_type']=df['cp_type'].astype(int)
 return df
def map_dose(df):
df['cp_dose']=df['cp_dose'].map({'D1': 1, 'D2': 0})
 df['cp_dose']=df['cp_dose'].astype(int)
 return df
def map_time(df):
df['cp_time']=df['cp_time'].map({24: 0, 48: 1, 72: 2})
 df['cp_time']=df['cp_time'].astype(int)
 return df<categorify>
train_X = cmb_df.iloc[:250]
def build_preprocess(preprocesses=[map_time,map_dose,map_controls]):
def preprocesser(df):
for proc in preprocesses:
df = proc(df)
 return df
return preprocesser 
<categorify>
test_X = cmb_df.iloc[250:]
preprocess_data=build_preprocess()<categorify>
shuffle_split = StratifiedShuffleSplit(test_size=0.25,train_size=0.75,n_splits=20 )
<categorify>
param_grid = {'n_estimators' : [1200],
'max_features' : [100,150],
'max_depth' : [5],
'class_weight' : [{1:0.5, 0:0.5} ,{1:0.6, 0:0.4},{1:0.4, 0:0.6}]
}
grid_search = GridSearchCV(
 estimator = RandomForestClassifier(random_state=12,n_jobs=-1),
param_grid = param_grid,
cv = shuffle_split,
return_train_score=True 
)
 grid_search.fit(train_X,target )
def multifold_indexer(train,target_columns,n_splits=10,random_state=12347,**kwargs):
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=n_splits,random_state=random_state,**kwargs)
 folds[ 'kfold']=0
for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=train[target_columns])) :
folds.iloc[v_idx,-1] = int(f)
 folds['kfold'] = folds['kfold'].astype(int)
 return folds 
<categorify>
print("Best parameters : {}".format(grid_search.best_params_))
print("Best cross validation score: {:.2f}".format(grid_search.best_score_))
print("Best estimator: {}".format(grid_search.best_estimator_))
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct 
<train_model>
results = pd.DataFrame(grid_search.cv_results_ )
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
if not scheduler.__class__ == torch.optim.lr_scheduler.ReduceLROnPlateau:
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, scheduler, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 if scheduler.__class__ == torch.optim.lr_scheduler.ReduceLROnPlateau:
scheduler.step(final_loss)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds 
<normalization>
scores_mean = np.array(results.mean_test_score ).reshape(-1)
 scores_std = np.array(results.std_test_score ).reshape(-1 )
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss<normalization>
print("mean CV scores for each fold {} ".format(scores_mean))
print("std CV scores for each fold {} ".format(scores_std))
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size,drop_rate1=0.2,drop_rate2=0.5,drop_rate3=0.8):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(drop_rate1)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(drop_rate2)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(drop_rate3)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x<feature_engineering>
pred_lr = grid_search.predict_proba(test_X)[:,1]
<define_variables><EOS> 
sub_df = pd.DataFrame()
sub_df["id"] = test_id
sub_df["target"] = pred_lr
sub_df.to_csv("baseline_lr.csv", index=False )