Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<import_modules>
warnings.simplefilter('ignore' )
class EasyDict(dict):
def __init__(self, d=None, **kwargs):
if d is None:
d = {}
if kwargs:
d.update(**kwargs)
 for k, v in d.items() :
setattr(self, k, v)
 for k in self.__class__.__dict__.keys() :
if not(k.startswith('__')and k.endswith('__')) and not k in('update', 'pop'):
setattr(self, k, getattr(self, k))
def __setattr__(self, name, value):
if isinstance(value,(list, tuple)) :
value = [self.__class__(x)
 if isinstance(x, dict)else x for x in value]
elif isinstance(value, dict)and not isinstance(value, self.__class__):
value = self.__class__(value)
 super(EasyDict, self ).__setattr__(name, value)
 super(EasyDict, self ).__setitem__(name, value)
 __setitem__ = __setattr__
def update(self, e=None, **f):
d = e or dict()
d.update(f)
 for k in d:
setattr(self, k, d[k])
 def pop(self, k, d=None):
delattr(self, k)
 return super(EasyDict, self ).pop(k, d )<normalization>
root = './'
id_name = 'sig_id'
variance_threshould = 0.7
ncompo_genes = 80
ncompo_cells = 10
seed=817119
def read_audio(conf, pathname, trim_long_data):
y, sr = librosa.load(pathname, sr=conf.sampling_rate)
 if 0 < len(y):
y, _ = librosa.effects.trim(y)
 if len(y)> conf.samples:
if trim_long_data:
y = y[0:0+conf.samples]
else:
padding = conf.samples - len(y)
 offset = padding // 2
y = np.pad(y,(offset, conf.samples - len(y)- offset), 'constant')
 return y
def audio_to_melspectrogram(conf, audio):
spectrogram = librosa.feature.melspectrogram(audio,
sr=conf.sampling_rate,
n_mels=conf.n_mels,
hop_length=conf.hop_length,
n_fft=conf.n_fft,
fmin=conf.fmin,
fmax=conf.fmax)
 spectrogram = librosa.power_to_db(spectrogram)
 spectrogram = spectrogram.astype(np.float32)
 return spectrogram
def melspectrogram_to_delta(mels):
return librosa.feature.delta(mels)
 def show_melspectrogram(conf, mels, title='Log-frequency power spectrogram'):
librosa.display.specshow(mels, x_axis='time', y_axis='mel',
sr=conf.sampling_rate, hop_length=conf.hop_length,
fmin=conf.fmin, fmax=conf.fmax)
 plt.colorbar(format='%+2.0f dB')
 plt.title(title)
 plt.show()
def read_as_melspectrogram(conf, pathname, trim_long_data, debug_display=False):
x = read_audio(conf, pathname, trim_long_data)
 mels = audio_to_melspectrogram(conf, x)
 if debug_display:
delta = melspectrogram_to_delta(mels)
 delta_squared = melspectrogram_to_delta(delta)
 IPython.display.display(IPython.display.Audio(x, rate=conf.sampling_rate))
show_melspectrogram(conf, mels)
 show_melspectrogram(conf, delta)
 show_melspectrogram(conf, delta_squared)
 return mels
conf = EasyDict()
conf.sampling_rate = 44100
conf.duration = 2
conf.hop_length = 347 * conf.duration
conf.fmin = 20
conf.fmax = conf.sampling_rate // 2
conf.n_mels = 128
conf.n_fft = conf.n_mels * 20
conf.samples = conf.sampling_rate * conf.duration<normalization>
def Seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
Seed_everything(seed=42 )
path = '.. /input/train_curated/0006ae4e.wav'
x = read_audio(conf, path, trim_long_data=False)
 print(x)
 print('_'*40)
 print(audio_to_melspectrogram(conf, x))
print(audio_to_melspectrogram(conf, x ).shape)
 x1 = read_as_melspectrogram(conf, path, trim_long_data=False, debug_display=True )<prepare_x_and_y>
def Metric(labels,preds):
labels = np.array(labels)
 preds = np.array(preds)
 metric = 0
for i in range(labels.shape[1]):
metric +=(-np.mean(labels[:,i]*np.log(np.maximum(preds[:,i],1e-15)) +(1-labels[:,i])*np.log(np.maximum(1-preds[:,i],1e-15))))
return metric/labels.shape[1]
def Write_log(logFile,text,isPrint=True):
if isPrint:
print(text)
 logFile.write(text)
 logFile.write('
')
 return None
def Rank_gauss_encoding(df,cols):
for col in cols:
print('rank gauss encoding:',col)
 tmp = df[col].rank().values - 1.0
tmp =(tmp / tmp.max())* 0.998 + 0.001
efi = np.sqrt(2.0)*erfinv(tmp)
 efi = efi - efi.mean()
df[col] = np.float16(efi)
 return df
def rationalApproximation(t):
c = [2.515517, 0.802853, 0.010328]
d = 1.432788, 0.189269, 0.001308
return t -(( c[2]*t + c[1])*t + c[0])/(((d[2]*t + d[1])*t + d[0])*t + 1.0)
 def normalCDFInverse(p):
if(p <= 0.0 or p >= 1.0):
assert(False)
 if(p < 0.5):
return -rationalApproximation(np.sqrt(-2.0*np.log(p)))
 return rationalApproximation(np.sqrt(-2.0*np.log(1-p)))
 def vdErfInvSingle(x):
if x == 0.0:
return 0.0
elif x < 0.0:
return -normalCDFInverse(-x)*0.7
else:
return normalCDFInverse(x)*0.7
def rankGauss(df, features):
df_size = df.shape[0]
for f in features:
if len(set(df[f])) == 1:
df[f] = 0
elif len(set(df[f])) == 2:
vals = sorted(list(set(df[f])))
 df[f] = df[f].replace(vals,[0,1])
 else:
df["rank"] =(df[f].rank(method="min")-1.0)/df_size*0.998+0.001
df[f] = df["rank"].apply(lambda x:vdErfInvSingle(x))
df[f] = df[f] - df[f].mean()
print("feature:%s rankGauss over"%(f))
if "rank" in df.columns.values.tolist() :
df.drop(["rank"],axis=1,inplace=True)
 print("rankGauss over")
 return df
training_curated_df, X_train_curated = convert_wav_to_image(training_curated_df, source=Path('.. /input/train_curated'), img_dest=Path('trn_curated'))
testing_df, X_test = convert_wav_to_image(testing_df, source=Path('.. /input/test'), img_dest=Path('test'))
print(f"Finished data conversion at {(time.time() -start_time)/3600} hours" )<load_from_csv>
files = ['.. /input/lish-moa/test_features.csv',
'.. /input/lish-moa/train_targets_scored.csv',
'.. /input/lish-moa/train_features.csv',
'.. /input/lish-moa/train_targets_nonscored.csv',
'.. /input/lish-moa/train_drug.csv',
'.. /input/lish-moa/sample_submission.csv']
test = pd.read_csv(files[0])
 train_target = pd.read_csv(files[1])
 train = pd.read_csv(files[2])
 train_nonscored = pd.read_csv(files[3])
 train_drug = pd.read_csv(files[4])
 sub = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
bs = 64
size = 128
tfms = get_transforms(do_flip=False, max_rotate=0, max_lighting=0.1, max_zoom=0, max_warp=0.)
src =(ImageList.from_csv('/kaggle/working', '.. /input/train_curated.csv', folder='.. /input/train_curated')
.split_by_rand_pct(0.2 ).label_from_df(label_delim=','))
data = src.transform(tfms, size=size ).databunch(bs=bs ).normalize()<choose_model_class>
genes = [col for col in train.columns if col.startswith("g-")]
cells = [col for col in train.columns if col.startswith("c-")]
features = genes + cells
targets = [col for col in train_target if col!='sig_id']
arch = models.resnet18
learn = cnn_learner(data, arch, pretrained=False, metrics=[lwlrap], wd = 0.1, ps = 0.5)
 learn.lr_find()
learn.recorder.plot()<train_model>
ori_train = train.copy()
ctl_train = train.loc[train['cp_type']=='ctl_vehicle'].append(test.loc[test['cp_type']=='ctl_vehicle'] ).reset_index(drop=True)
 ctl_train2 = train.loc[train['cp_type']=='ctl_vehicle'].reset_index(drop=True)
 ori_test = test.copy()
ctl_test = test.loc[test['cp_type']=='ctl_vehicle'].reset_index(drop=True )
learn.fit_one_cycle(5, 1e-2)
 learn.save('first-attempt-128' )<train_model>
def Feature(df):
for col in ['cp_time','cp_dose']:
tmp = pd.get_dummies(df[col],prefix=col)
 df = pd.concat([df,tmp],axis=1)
 df.drop([col],axis=1,inplace=True)
 for col in genes:
if df[col].std() < 1.0:
df.drop([col],axis=1,inplace=True)
 print(col)
 genes.remove(col)
 df[genes+cells] = df[genes+cells]/10.0
df['gene_gt_0'] =(df[genes]>0.).mean(axis=1)
 df['gene_lt_0'] =(df[genes]<0.).mean(axis=1)
 df['cell_gt_0'] =(df[cells]>0.).mean(axis=1)
 df['cell_lt_0'] =(df[cells]<0.).mean(axis=1)
 df['gene_gt_0.7'] =(df[genes]>0.7 ).mean(axis=1)
 df['gene_lt_0.7'] =(df[genes]<0.7 ).mean(axis=1)
 df['cell_gt_0.7'] =(df[cells]>0.7 ).mean(axis=1)
 df['cell_lt_0.7'] =(df[cells]<0.7 ).mean(axis=1)
 df['gene_gt_0.9'] =(df[genes]>0.9 ).mean(axis=1)
 df['gene_lt_0.9'] =(df[genes]<0.9 ).mean(axis=1)
 df['cell_gt_0.9'] =(df[cells]>0.9 ).mean(axis=1)
 df['cell_lt_0.9'] =(df[cells]<0.9 ).mean(axis=1)
 pca_genes = PCA(n_components = ncompo_genes,
random_state = seed ).fit_transform(df[genes])
 pca_cells = PCA(n_components = ncompo_cells,
random_state = seed ).fit_transform(df[cells])
 pca_genes = pd.DataFrame(pca_genes, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells = pd.DataFrame(pca_cells, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 df = pd.concat([df, pca_genes, pca_cells], axis = 1)
 pca_cols = [col for col in df.columns if 'pca' in col]
for col in pca_cols:
df[col] = df[col] / df[col].abs().max()
return df
learn.unfreeze()
learn.fit_one_cycle(1 )<train_model>
def Feature1(df):

 for col in tqdm(genes+cells):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution='normal')
 transformer.fit(df[:train.shape[0]][col].values.reshape(-1,1))
df[col] = transformer.transform(df[col].values.reshape(-1,1)).reshape(1,-1)[0]
pca_genes = PCA(n_components = ncompo_genes,
random_state = 42 ).fit_transform(df[genes])
 pca_cells = PCA(n_components = ncompo_cells,
random_state = 42 ).fit_transform(df[cells])
 pca_genes = pd.DataFrame(pca_genes, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells = pd.DataFrame(pca_cells, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 df = pd.concat([df, pca_genes, pca_cells], axis = 1)
 nor_var_col = [col for col in df.columns if col in ['sig_id','cp_type','cp_time','cp_dose'] or '_gt_' in col or '_lt_' in col]
var_thresh = VarianceThreshold(0.8)
 var_cols = [col for col in df.columns if col not in ['sig_id','cp_type','cp_time','cp_dose'] and '_gt_' not in col and '_lt_' not in col]
var_data = var_thresh.fit_transform(df[var_cols])
 df = pd.concat([df[nor_var_col],pd.DataFrame(var_data)],axis=1)

for col in ['cp_time','cp_dose']:
tmp = pd.get_dummies(df[col],prefix=col)
 df = pd.concat([df,tmp],axis=1)
 df.drop([col],axis=1,inplace=True)
 return df
learn.fit(20, slice(2e-3, 2e-4))
learn.save('second-attempt-128' )<train_model>
def Feature2(df):

 for col in tqdm(genes+cells):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution='normal')
 transformer.fit(df[:train.shape[0]][col].values.reshape(-1,1))
df[col] = transformer.transform(df[col].values.reshape(-1,1)).reshape(1,-1)[0]
gene_pca = PCA(n_components = ncompo_genes,
random_state = 42 ).fit(df[genes])
 pca_genes = gene_pca.transform(df[genes])
 cell_pca = PCA(n_components = ncompo_cells,
random_state = 42 ).fit(df[cells])
 pca_cells = cell_pca.transform(df[cells])
 pca_genes = pd.DataFrame(pca_genes, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells = pd.DataFrame(pca_cells, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 df = pd.concat([df, pca_genes, pca_cells], axis = 1)

for col in ['cp_time','cp_dose']:
tmp = pd.get_dummies(df[col],prefix=col)
 df = pd.concat([df,tmp],axis=1)
 df.drop([col],axis=1,inplace=True)
 return df,transformer,gene_pca,cell_pca
learn.fit(20, slice(1e-3, 1e-4))
learn.save('third-attempt-128' )<categorify>
tt = train.append(test ).reset_index(drop=True)
 tt,transformer,gene_pca,cell_pca = Feature2(tt)
 train = tt[:train.shape[0]]
test = tt[train.shape[0]:].reset_index(drop=True )
size = 256
data = src.transform(tfms, size=size ).databunch(bs=bs ).normalize(imagenet_stats )<train_model>
if 1:
train_target = train_target.loc[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 ori_train = ori_train.loc[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 train = train.loc[train['cp_type']!='ctl_vehicle'].reset_index(drop=True )
learn.data = data
learn.freeze()
learn.lr_find()
learn.recorder.plot()<train_model>
class resnetModel(nn.Module):
def __init__(self, num_features, num_targets, hidden_size,ispretrain=False):
super(resnetModel, self ).__init__()
self.ispretrain=ispretrain
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(num_features+hidden_size)
 self.dropout2 = nn.Dropout(0.2619422201258426)
 self.dense2 = nn.utils.weight_norm(nn.Linear(num_features+hidden_size, hidden_size))
self.batch_norm20 = nn.BatchNorm1d(hidden_size)
 self.dropout20 = nn.Dropout(0.2619422201258426)
 self.dense20 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(2*hidden_size)
 self.dropout3 = nn.Dropout(0.2619422201258426)
 self.dense3 = nn.utils.weight_norm(nn.Linear(2*hidden_size, hidden_size))
self.batch_norm30 = nn.BatchNorm1d(hidden_size)
 self.dropout30 = nn.Dropout(0.2619422201258426)
 self.dense30 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm4 = nn.BatchNorm1d(2*hidden_size)
 self.dropout4 = nn.Dropout(0.2619422201258426)
 if self.ispretrain:
self.dense4 = nn.utils.weight_norm(nn.Linear(2*hidden_size, num_targets))
else:
self.dense5 = nn.utils.weight_norm(nn.Linear(2*hidden_size, num_targets))
def forward(self, x):
x1 = self.batch_norm1(x)
 x1 = F.leaky_relu(self.dense1(x1))
x = torch.cat([x,x1],1)
 x2 = self.batch_norm2(x)
 x2 = self.dropout2(x2)
 x2 = F.leaky_relu(self.dense2(x2))
x2 = self.batch_norm20(x2)
 x2 = self.dropout20(x2)
 x2 = F.leaky_relu(self.dense20(x2))
x = torch.cat([x1,x2],1)
 x3 = self.batch_norm3(x)
 x3 = self.dropout3(x3)
 x3 = F.leaky_relu(self.dense3(x3))
x3 = self.batch_norm30(x3)
 x3 = self.dropout30(x3)
 x3 = F.leaky_relu(self.dense30(x3))
x3 = torch.cat([x2,x3],1)
 x3 = self.batch_norm4(x3)
 x3 = self.dropout4(x3)
 if self.ispretrain:
x3 = self.dense4(x3)
 else:
x3 = self.dense5(x3)
 return x3
learn.fit(5, 3e-3)
 learn.save('first-attempt-256' )<find_best_params>
class resnetModel2(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(resnetModel, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(num_features+hidden_size)
 self.dropout2 = nn.Dropout(0.2619422201258426)
 self.dense2 = nn.utils.weight_norm(nn.Linear(num_features+hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(2*hidden_size)
 self.dropout3 = nn.Dropout(0.2619422201258426)
 self.dense3 = nn.utils.weight_norm(nn.Linear(2*hidden_size, hidden_size))
self.batch_norm4 = nn.BatchNorm1d(2*hidden_size)
 self.dropout4 = nn.Dropout(0.2619422201258426)
 self.dense4 = nn.utils.weight_norm(nn.Linear(2*hidden_size, num_targets))
def forward(self, x):
x1 = self.batch_norm1(x)
 x1 = F.leaky_relu(self.dense1(x1))
x = torch.cat([x,x1],1)
 x2 = self.batch_norm2(x)
 x2 = self.dropout2(x2)
 x2 = F.leaky_relu(self.dense2(x2))
x = torch.cat([x1,x2],1)
 x3 = self.batch_norm3(x)
 x3 = self.dropout3(x3)
 x3 = self.dense3(x3)
 x3 = torch.cat([x2,x3],1)
 x3 = self.batch_norm4(x3)
 x3 = self.dropout4(x3)
 x3 = self.dense4(x3)
 return x3
learn.unfreeze()
learn.lr_find()
learn.recorder.plot()<train_model>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
learn.fit(7, slice(1e-3, 1e-4))
learn.save('second-attempt-256' )<train_model>
def Ctl_augment(train,target,include_test=0):
if include_test==0:
ctl_aug=ctl_train2.copy()
if include_test==1:
ctl_aug=ctl_train.copy()
aug_trains = []
aug_targets = []
for t in [24,48,72]:
for d in ['D1','D2']:
for _ in range(3):
train1 = train.loc[(train['cp_time']==t)&(train['cp_dose']==d)]
target1 = target.loc[(train['cp_time']==t)&(train['cp_dose']==d)]
ctl1 = ctl_aug.loc[(ctl_aug['cp_time']==t)&(ctl_aug['cp_dose']==d)].sample(train1.shape[0],replace=True)
 ctl2 = ctl_aug.loc[(ctl_aug['cp_time']==t)&(ctl_aug['cp_dose']==d)].sample(train1.shape[0],replace=True)
 train1[genes+cells] = train1[genes+cells].values + ctl1[genes+cells].values - ctl2[genes+cells].values
aug_train = train1.merge(target1,how='left',on='sig_id')
 aug_trains.append(aug_train[['cp_time','cp_dose']+genes+cells])
 aug_targets.append(aug_train[targets])
 df = pd.concat(aug_trains ).reset_index(drop=True)
 target = pd.concat(aug_targets ).reset_index(drop=True)
 for col in tqdm(genes+cells):
df[col] = transformer.transform(df[col].values.reshape(-1,1)).reshape(1,-1)[0]
pca_genes = gene_pca.transform(df[genes])
 pca_cells = cell_pca.transform(df[cells])
 pca_genes = pd.DataFrame(pca_genes, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells = pd.DataFrame(pca_cells, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 df = pd.concat([df, pca_genes, pca_cells], axis = 1)
 for col in ['cp_time','cp_dose']:
tmp = pd.get_dummies(df[col],prefix=col)
 df = pd.concat([df,tmp],axis=1)
 df.drop([col],axis=1,inplace=True)
 xs = df[train_cols].values
ys = target[targets]
return xs,ys
learn.fit_one_cycle(20, slice(5e-4, 5e-5), callbacks=[SaveModelCallback(learn, monitor='lwlrap', mode='max')] )<save_to_csv>
class MoADataset:
def __init__(self, features, targets,noise=0.1,val=0):
self.features = features
self.targets = targets
self.noise = noise
self.val = val
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
sample = self.features[idx, :].copy()
if 0 and np.random.rand() <0.3 and not self.val:
sample = self.swap_sample(sample)
 dct = {
'x' : torch.tensor(sample, dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
def swap_sample(self,sample):
num_samples = self.features.shape[0]
num_features = self.features.shape[1]
if len(sample.shape)== 2:
batch_size = sample.shape[0]
random_row = np.random.randint(0, num_samples, size=batch_size)
 for i in range(batch_size):
random_col = np.random.rand(num_features)< self.noise
sample[i, random_col] = self.features[random_row[i], random_col]
else:
batch_size = 1
random_row = np.random.randint(0, num_samples, size=batch_size)
 random_col = np.random.rand(num_features)< self.noise
sample[ random_col] = self.features[random_row, random_col]
return sample
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
CUR_X_FILES, CUR_X = list(testing_df.fname.values), X_test
test = ImageList.from_csv(Path('/kaggle/working'), Path('.. /input/sample_submission.csv'), folder=Path('.. /input/test'))
learn = load_learner(Path('/kaggle/working'), test=test)
 preds, _ = learn.get_preds(ds_type=DatasetType.Test)
 testing_df[learn.data.classes] = preds
testing_df.to_csv('submission.csv', index=False)
 testing_df.head()<import_modules>
cs_df = pd.read_csv('.. /input/moa-cv-file/cv_fold.csv')
 cs_df=cs_df[cs_df['sig_id'].isin(train.sig_id)].reset_index(drop=True)
 device =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS1 = 29
EPOCHS = 23
trn_loss_=[]
def train_and_predict(features, sub, aug, folds=5, seed=817119,lr=1/90.0/3.5*3,weight_decay=1e-5/3):
dnn_oof = train[['sig_id']]
oof = train[['sig_id']]
for t in targets:
dnn_oof[t] = 0.0
oof[t] = 0.0
preds = []
test_X = test[features].values
test_data_loader = DataLoader(dataset=TensorDataset(torch.Tensor(test_X)) ,batch_size=1024,shuffle=False)
 for fold,(trn_ind, val_ind)in enumerate(StratifiedKFold(n_splits = folds, shuffle=True, random_state=seed)\ 
.split(train, cs_df['most_cs_same_target'])) :
train_X = train.loc[trn_ind,features].values
train_Y = train_target.loc[trn_ind,targets].values
train_Y = train_target.loc[trn_ind,targets].values
if 0:
aug_X,aug_Y = Ctl_augment(ori_train.loc[trn_ind],train_target.loc[trn_ind],include_test=1)
 train_X = np.concatenate([train_X,aug_X],axis=0)
 train_Y = np.concatenate([train_Y,aug_Y],axis=0)
 valid_X = train.loc[val_ind,features].values
valid_Y = train_target.loc[val_ind,targets].values

 valid_dataset = MoADataset(valid_X, valid_Y,val=1)
 valid_data_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=1024, shuffle=False)
 model = resnetModel(len(features),len(targets),1500)
 model.to(device)
 aug_X,aug_Y = Ctl_augment(ori_train.loc[trn_ind],train_target.loc[trn_ind],include_test=1)
 train_X_ = np.concatenate([train_X,aug_X],axis=0)
 train_Y_ = np.concatenate([train_Y,aug_Y],axis=0)
 train_dataset = MoADataset(train_X_, train_Y_)
 train_data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=128, shuffle=True)
 optimizer = torch.optim.Adam(model.parameters() ,betas=(0.9, 0.99), lr=1e-3, weight_decay=weight_decay,eps=1e-5)
 scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=lr, epochs=EPOCHS1, steps_per_epoch=len(train_data_loader))
loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 best_valid_metric = 1e9
not_improve_epochs = 0
for epoch in range(EPOCHS1):
train_loss = 0.0
train_num = 0
for data in(train_data_loader):
optimizer.zero_grad()
x,y = data['x'].to(device),data['y'].to(device)
 outputs = model(x)
 loss = loss_tr(outputs, y)
 loss.backward()
optimizer.step()
scheduler.step()
train_num += x.shape[0]
train_loss +=(loss.item() *x.shape[0])
 train_loss /= train_num
model.eval()
valid_loss = 0.0
valid_num = 0
for data in(valid_data_loader):
x,y = data['x'].to(device),data['y'].to(device)
 outputs = model(x)
 loss = loss_fn(outputs, y)
 valid_num += x.shape[0]
valid_loss +=(loss.item() *x.shape[0])
 valid_loss /= valid_num
t_preds = []
for data in(test_data_loader):
x = data[0].to(device)
 with torch.no_grad() :
outputs = model(x)
 t_preds.extend(list(outputs.sigmoid().cpu().detach().numpy()))
pred_mean = np.mean(t_preds)
 if valid_loss < best_valid_metric:
torch.save(model.state_dict() ,'./model_resnet_fold%s.ckpt'%fold)
 not_improve_epochs = 0
best_valid_metric = valid_loss
print('[epoch %s] lr: %.6f, train_loss: %.6f, valid_metric: %.6f, pred_mean:%.6f'%(epoch,optimizer.param_groups[0]['lr'],train_loss,valid_loss,pred_mean))
trn_loss_.append(train_loss)
 else:
not_improve_epochs += 1
print('[epoch %s] lr: %.6f, train_loss: %.6f, valid_metric: %.6f, pred_mean:%.6f, NIE +1 ---> %s'%(epoch,optimizer.param_groups[0]['lr'],train_loss,valid_loss,pred_mean,not_improve_epochs))
if not_improve_epochs >= 50:
break
model.train()
if epoch!=EPOCHS1-1:
aug_X,aug_Y = Ctl_augment(ori_train.loc[trn_ind],train_target.loc[trn_ind],include_test=1)
 train_X_ = np.concatenate([train_X,aug_X],axis=0)
 train_Y_ = np.concatenate([train_Y,aug_Y],axis=0)
 train_dataset = MoADataset(train_X_, train_Y_)
 train_data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=128, shuffle=True)
 model = resnetModel(len(features),len(targets),1500)
 model.to(device)
 state_dict = torch.load('./model_resnet_fold%s.ckpt'%fold, torch.device("cuda" if torch.cuda.is_available() else "cpu"))
model.load_state_dict(state_dict)
 model.eval()
valid_preds = []
for data in tqdm(valid_data_loader):
x,y = data['x'].to(device),data['y'].to(device)
 with torch.no_grad() :
outputs = model(x)
 valid_preds.extend(list(outputs.cpu().detach().numpy()))
dnn_oof.loc[val_ind,targets] = 1 /(1+np.exp(-np.array(valid_preds)))
 t_preds = []
for data in tqdm(test_data_loader):
x = data[0].to(device)
 with torch.no_grad() :
outputs = model(x)
 t_preds.extend(list(outputs.sigmoid().cpu().detach().numpy()))
print(np.mean(t_preds))
preds.append(t_preds)
 sub[targets] = np.array(preds ).mean(axis=0)
 return dnn_oof,oof,sub
import gc
import os
import pickle
import random
import time
from collections import Counter, defaultdict
from functools import partial
from pathlib import Path
from psutil import cpu_count
import matplotlib.pyplot as plt
import librosa
import numpy as np
import pandas as pd
from PIL import Image
from sklearn.model_selection import train_test_split
from imgaug import augmenters as iaa
import torch
import torch.nn as nn
import torch.nn.functional as F
from fastprogress import master_bar, progress_bar
from torch.optim import Adam
from torch.optim.lr_scheduler import CosineAnnealingLR
from torch.utils.data import Dataset, DataLoader
from torchvision.transforms import transforms<define_variables>
train_cols = [col for col in train.columns if col not in ['sig_id','cp_type','cs','most_cs_ind','most_cs_same_target']]
len(train_cols )
NUM_CLASSES = 80
SIZE=128
checkpoint_file = ['model_best1.h5', 'model_best2.h5', 'model_best3.h5']
EPOCHS = [432, 0, 0]
TTA = [19, 0, 0]
BATCH_SIZE = 32
LR = 4e-4
PATIENCE = 10
LR_FACTOR = 0.8
CURATED_ONLY = True
TRAIN_AUGMENT = True
VALID_AUGMENT = False
MODEL = 'mobile'
SEED = 520
USE_MIXUP = True
MIXUP_PROB = 0.275
ACTIVATION = 'linear'
LOSS = 'BCEwithLogits'<set_options>
Seed_everything(817119)
 dnn_oof,oof,sub = train_and_predict(train_cols,sub.copy() ,aug=True,seed=817119,lr=1/90.0/3.5*4,weight_decay=1e-5/4)
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(SEED )<define_variables>
valid_metric = Metric(train_target[targets].values,dnn_oof[targets].values)
def _one_sample_positive_class_precisions(scores, truth):

 num_classes = scores.shape[0]
pos_class_indices = np.flatnonzero(truth > 0)
 if not len(pos_class_indices):
return pos_class_indices, np.zeros(0)
 retrieved_classes = np.argsort(scores)[::-1]
class_rankings = np.zeros(num_classes, dtype=np.int)
 class_rankings[retrieved_classes] = range(num_classes)
 retrieved_class_true = np.zeros(num_classes, dtype=np.bool)
 retrieved_class_true[class_rankings[pos_class_indices]] = True
retrieved_cumulative_hits = np.cumsum(retrieved_class_true)
 precision_at_hits =(
 retrieved_cumulative_hits[class_rankings[pos_class_indices]] / 
(1 + class_rankings[pos_class_indices].astype(np.float)))
 return pos_class_indices, precision_at_hits
def calculate_per_class_lwlrap(truth, scores):

 assert truth.shape == scores.shape
num_samples, num_classes = scores.shape
precisions_for_samples_by_classes = np.zeros(( num_samples, num_classes))
for sample_num in range(num_samples):
pos_class_indices, precision_at_hits =(
 _one_sample_positive_class_precisions(scores[sample_num, :],
truth[sample_num, :]))
precisions_for_samples_by_classes[sample_num, pos_class_indices] =(
 precision_at_hits)
 labels_per_class = np.sum(truth > 0, axis=0)
 weight_per_class = labels_per_class / float(np.sum(labels_per_class))
per_class_lwlrap =(np.sum(precisions_for_samples_by_classes, axis=0)/
np.maximum(1, labels_per_class))
return per_class_lwlrap, weight_per_class<define_variables>
sub.loc[test['cp_type']=='ctl_vehicle',targets] = 0.0
<compute_test_metric><EOS> 
dnn_oof.to_csv('./oof.csv',index=False)
 sub.to_csv('./submission.csv',index=False )