Kaggle competition: Generative Dog Images
<SOS> metric: PostProcessorKernel Kaggle data source: generative-dog-images<save_to_csv>
%matplotlib inline
warnings.filterwarnings('ignore', category=FutureWarning)
 device = 'cuda' if torch.cuda.is_available() else 'cpu'
submission.to_csv('submission.csv', index=False )<save_to_csv>
config = {'DataLoader': {'batch_size': 64,
'shuffle': True},
'Generator': {'latent_dim': 120,
'embed_dim': 32,
'ch': 64,
'num_classes': 120,
'use_attn': True},
'Discriminator': {'ch': 64,
'num_classes': 120,
'use_attn': True},
'sample_latents': {'latent_dim': 120,
'num_classes': 120},
'num_iterations': 50000,
'decay_start_iteration': 50000,
'd_steps': 1,
'lr_G': 2e-4,
'lr_D': 4e-4,
'betas':(0.0, 0.999),
'margin': 1.0,
'gamma': 0.1,
'ema': 0.999,
'seed': 42}
submission.to_csv('submission.csv', index=False )<import_modules>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(config['seed'] )
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import roc_auc_score, make_scorer
from scipy.stats import describe, rankdata, mannwhitneyu
from tqdm import tqdm_notebook
import os 
<load_from_csv>
root_images = '.. /input/all-dogs/all-dogs/'
root_annots = '.. /input/annotation/Annotation/'
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<load_from_csv>
all_files = os.listdir(root_images )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<sort_values>
breeds = glob.glob(root_annots+'*')
 annotations = []
for breed in breeds:
annotations += glob.glob(breed+'/*' )
corrs = train.corr().abs().unstack().sort_values(kind="quicksort" ).reset_index()
corrs = corrs[corrs['level_0'] != corrs['level_1']]
corrs.tail(10 )<prepare_x_and_y>
breed_map = {}
for annotation in annotations:
breed = annotation.split('/')[-2]
index = breed.split('-')[0]
breed_map.setdefault(index, breed )
y = train['target']
train.drop('target', axis = 1, inplace = True )<feature_engineering>
all_labels = [breed_map[file.split('_')[0]] for file in all_files]
le = LabelEncoder()
all_labels = le.fit_transform(all_labels )
mw = pd.DataFrame(index = ['stat', 'p'])
 for c in train.columns:
mw[c] = mannwhitneyu(train[c], test[c])
 mw = mw.T
mw[mw['p'] <.01].shape
bad_features = list(mw[mw['p'] <.01].index)
 print(bad_features )<normalization>
all_bboxes = [load_bbox(file)for file in all_files]
def apply_noise(data, noise_level):
idxt = data[['id', 'target']]
features = data.columns.values[2:]
appended_data = []
for feature in features:
signal = data[feature]
noise_factor =(np.abs(signal)).mean() * noise_level
noise = np.random.normal(0, noise_level, signal.shape)
 jittered = signal + noise
appended_data.append(pd.DataFrame(jittered))
appended_data = pd.concat(appended_data, axis=1)
 data_jittered = pd.concat([idxt, pd.DataFrame(appended_data)], axis=1)
 return data_jittered
noise_train_df = [] 
<split>
print('Total files : {}'.format(len(all_files)))
 print('Total labels : {}'.format(len(all_labels)))
 print('Total bboxes : {}'.format(len(all_bboxes)))
 print('Total annotations : {}'.format(len(annotations)))
 print('Total classes : {}'.format(len(le.classes_)) )
X = train.drop(bad_features, axis=1)
<compute_train_metric>
all_images = [load_bboxcrop_resized_image(f, b)for f, b in zip(all_files, resized_bboxes)]
all_images = np.array(all_images )
lr = LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear')
 scores = cross_val_score(lr, X, y, scoring='roc_auc', cv=3)
 scores.mean()<save_to_csv>
class DogDataset(Dataset):
def __init__(self, images, labels):
super().__init__()
self.images = images
self.labels = labels
self.transform = A.Compose([A.HorizontalFlip(p=0.5), ToTensor() ])
 def __len__(self):
return len(self.images)
 def __getitem__(self, idx):
img, label = self.images[idx], self.labels[idx]
img = self.transform(image=img)['image']
label = torch.as_tensor(label, dtype=torch.long)
 return img, label
lr.fit(X, y)
 sub_pred = lr.predict_proba(test.drop(bad_features, axis=1)) [:,1]
sub = pd.DataFrame({"id":test["id"].values})
 sub["target"] = sub_pred
sub.to_csv("submission_prob_whitney.csv", index=False )<set_options>
def get_dataiterator(images, labels, dataloader_params, device='cpu'):
train_dataset = DogDataset(images, labels)
 train_dataloader = DataLoader(train_dataset, **dataloader_params)
 while True:
for imgs, labels in train_dataloader:
imgs, labels = imgs.to(device), labels.to(device)
 imgs +=(1.0 / 128.0)* torch.rand_like(imgs)
 yield imgs, labels
pd.set_option('max_columns', None)
 %matplotlib inline
py.init_notebook_mode(connected=True)
 warnings.filterwarnings('ignore')
<load_from_csv>
class Attention(nn.Module):
def __init__(self, channels, reduction_attn=8, reduction_sc=2):
super().__init__()
self.channles_attn = channels // reduction_attn
self.channels_sc = channels // reduction_sc
self.conv_query = spectral_norm(nn.Conv2d(channels, self.channles_attn, kernel_size=1, bias=False))
self.conv_key = spectral_norm(nn.Conv2d(channels, self.channles_attn, kernel_size=1, bias=False))
self.conv_value = spectral_norm(nn.Conv2d(channels, self.channels_sc, kernel_size=1, bias=False))
self.conv_attn = spectral_norm(nn.Conv2d(self.channels_sc, channels, kernel_size=1, bias=False))
self.gamma = nn.Parameter(torch.zeros(1))
nn.init.orthogonal_(self.conv_query.weight.data)
 nn.init.orthogonal_(self.conv_key.weight.data)
 nn.init.orthogonal_(self.conv_value.weight.data)
 nn.init.orthogonal_(self.conv_attn.weight.data)
 def forward(self, x):
batch, _, h, w = x.size()
proj_query = self.conv_query(x ).view(batch, self.channles_attn, -1)
 proj_key = F.max_pool2d(self.conv_key(x), 2 ).view(batch, self.channles_attn, -1)
 attn = torch.bmm(proj_key.permute(0,2,1), proj_query)
 attn = F.softmax(attn, dim=1)
 proj_value = F.max_pool2d(self.conv_value(x), 2 ).view(batch, self.channels_sc, -1)
 attn = torch.bmm(proj_value, attn)
 attn = attn.view(batch, self.channels_sc, h, w)
 attn = self.conv_attn(attn)
 out = self.gamma * attn + x
return out
%%time
train = pd.read_csv(os.path.join(location, 'train.csv'))
test = pd.read_csv(os.path.join(location, 'test.csv'))
sample_submission = pd.read_csv(os.path.join(location, 'sample_submission.csv'))<feature_engineering>
class CBN2d(nn.Module):
def __init__(self, num_features, num_conditions):
super().__init__()
self.bn = nn.BatchNorm2d(num_features, affine=False)
 self.embed = spectral_norm(nn.Conv2d(num_conditions, num_features*2, kernel_size=1, bias=False))
nn.init.orthogonal_(self.embed.weight.data)
 def forward(self, x, y):
out = self.bn(x)
 embed = self.embed(y.unsqueeze(2 ).unsqueeze(3))
gamma, beta = embed.chunk(2, dim=1)
 out =(1.0 + gamma)* out + beta
return out
def missing_data(data):
total = data.isnull().sum()
percent =(total/data.isnull().count() *100)
 miss_column = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 types = []
for col in data.columns:
dtype = str(data[col].dtype)
 types.append(dtype)
 miss_column['Types'] = types
return(np.transpose(miss_column))<count_missing_values>
class GBlock(nn.Module):
def __init__(self, in_channels, out_channels, num_conditions, upsample=False):
super().__init__()
self.upsample = upsample
self.learnable_sc = in_channels != out_channels or upsample
self.conv1 = spectral_norm(nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1, bias=False))
self.conv2 = spectral_norm(nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1, bias=False))
self.cbn1 = CBN2d(in_channels, num_conditions)
 self.cbn2 = CBN2d(out_channels, num_conditions)
 if self.learnable_sc:
self.conv_sc = spectral_norm(nn.Conv2d(in_channels, out_channels, kernel_size=1, bias=False))
self.relu = nn.ReLU()
nn.init.orthogonal_(self.conv1.weight.data)
 nn.init.orthogonal_(self.conv2.weight.data)
 if self.learnable_sc:
nn.init.orthogonal_(self.conv_sc.weight.data)
 def _upsample_conv(self, x, conv):
x = F.interpolate(x, scale_factor=2, mode='nearest')
 x = conv(x)
 return x
def _residual(self, x, y):
x = self.relu(self.cbn1(x, y))
x = self._upsample_conv(x, self.conv1)if self.upsample else self.conv1(x)
 x = self.relu(self.cbn2(x, y))
x = self.conv2(x)
 return x
def _shortcut(self, x):
if self.learnable_sc:
x = self._upsample_conv(x, self.conv_sc)if self.upsample else self.conv_sc(x)
 return x
def forward(self, x, y):
return self._shortcut(x)+ self._residual(x, y )
missing_data(train )<count_missing_values>
class Generator(nn.Module):
def __init__(self, latent_dim, ch, num_classes, embed_dim, use_attn=False):
super().__init__()
self.latent_dim = latent_dim
self.ch = ch
self.num_classes = num_classes
self.embed_dim = embed_dim
self.use_attn = use_attn
self.num_chunk = 5
num_latents = self.__get_num_latents()
self.embed = nn.Embedding(num_classes, embed_dim)
 self.fc = spectral_norm(nn.Linear(num_latents[0], ch*8*4*4, bias=False))
self.block1 = GBlock(ch*8, ch*8, num_latents[1], upsample=True)
 self.block2 = GBlock(ch*8, ch*4, num_latents[2], upsample=True)
 self.block3 = GBlock(ch*4, ch*2, num_latents[3], upsample=True)
 if use_attn:
self.attn = Attention(ch*2)
 self.block4 = GBlock(ch*2, ch, num_latents[4], upsample=True)
 self.bn = nn.BatchNorm2d(ch)
 self.relu = nn.ReLU()
self.conv_last = spectral_norm(nn.Conv2d(ch, 3, kernel_size=3, padding=1, bias=False))
self.tanh = nn.Tanh()
nn.init.orthogonal_(self.embed.weight.data)
 nn.init.orthogonal_(self.fc.weight.data)
 nn.init.orthogonal_(self.conv_last.weight.data)
 nn.init.constant_(self.bn.weight.data, 1.0)
 nn.init.constant_(self.bn.bias.data, 0.0)
 def __get_num_latents(self):
xs = torch.empty(self.latent_dim ).chunk(self.num_chunk)
 num_latents = [x.size(0)for x in xs]
for i in range(1, self.num_chunk):
num_latents[i] += self.embed_dim
return num_latents
def forward(self, x, y):
xs = x.chunk(self.num_chunk, dim=1)
 y = self.embed(y)
 h = self.fc(xs[0])
 h = h.view(h.size(0), self.ch*8, 4, 4)
 h = self.block1(h, torch.cat([y, xs[1]], dim=1))
h = self.block2(h, torch.cat([y, xs[2]], dim=1))
h = self.block3(h, torch.cat([y, xs[3]], dim=1))
if self.use_attn:
h = self.attn(h)
 h = self.block4(h, torch.cat([y, xs[4]], dim=1))
h = self.relu(self.bn(h))
out = self.tanh(self.conv_last(h))
return out
missing_data(test )<drop_column>
class DBlock(nn.Module):
def __init__(self, in_channels, out_channels, downsample=False, optimized=False):
super().__init__()
self.downsample = downsample
self.optimized = optimized
self.learnable_sc = in_channels != out_channels or downsample
self.conv1 = spectral_norm(nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1, bias=False))
self.conv2 = spectral_norm(nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1, bias=False))
if self.learnable_sc:
self.conv_sc = spectral_norm(nn.Conv2d(in_channels, out_channels, kernel_size=1, bias=False))
self.relu = nn.ReLU()
nn.init.orthogonal_(self.conv1.weight.data)
 nn.init.orthogonal_(self.conv2.weight.data)
 if self.learnable_sc:
nn.init.orthogonal_(self.conv_sc.weight.data)
 def _residual(self, x):
if not self.optimized:
x = self.relu(x)
 x = self.conv1(x)
 x = self.relu(x)
 x = self.conv2(x)
 if self.downsample:
x = F.avg_pool2d(x, 2)
 return x
def _shortcut(self, x):
if self.learnable_sc:
if self.optimized:
x = self.conv_sc(F.avg_pool2d(x, 2)) if self.downsample else self.conv_sc(x)
 else:
x = F.avg_pool2d(self.conv_sc(x), 2)if self.downsample else self.conv_sc(x)
 return x
def forward(self, x):
return self._shortcut(x)+ self._residual(x )
label = train['target']
train = train.drop(['id', 'target'], axis=1)
 test = test.drop(['id'], axis=1 )<count_values>
class Discriminator(nn.Module):
def __init__(self, ch, num_classes, use_attn=False):
super().__init__()
self.ch = ch
self.num_classes = num_classes
self.use_attn = use_attn
self.block1 = DBlock(3, ch, downsample=True, optimized=True)
 if use_attn:
self.attn = Attention(ch)
 self.block2 = DBlock(ch, ch*2, downsample=True)
 self.block3 = DBlock(ch*2, ch*4, downsample=True)
 self.block4 = DBlock(ch*4, ch*8, downsample=True)
 self.relu = nn.ReLU()
self.fc = spectral_norm(nn.Linear(ch*8, 1, bias=False))
self.embed = spectral_norm(nn.Embedding(num_classes, ch*8))
self.clf = spectral_norm(nn.Linear(ch*8, num_classes, bias=False))
nn.init.orthogonal_(self.fc.weight.data)
 nn.init.orthogonal_(self.embed.weight.data)
 nn.init.orthogonal_(self.clf.weight.data)
 def forward(self, x, y):
h = self.block1(x)
 if self.use_attn:
h = self.attn(h)
 h = self.block2(h)
 h = self.block3(h)
 h = self.block4(h)
 h = self.relu(h)
 h = torch.sum(h, dim=(2,3))
out = self.fc(h)
 out += torch.sum(self.embed(y)*h, dim=1, keepdim=True)
 ac = self.clf(h)
 ac = F.log_softmax(ac, dim=1)
 return out, ac
label_count = label.value_counts().reset_index().rename(columns = {'index' : 'Labels'})
 label_count<count_missing_values>
train_dataiterator = get_dataiterator(all_images, all_labels, config['DataLoader'], device=device )
missing_data(train )<split>
netG = Generator(**config['Generator'] ).to(device, torch.float32)
 netD = Discriminator(**config['Discriminator'] ).to(device, torch.float32 )
folds = StratifiedKFold(n_splits=10, shuffle=True, random_state=4422)
 forcast_rfc2 = np.zeros(len(xtest))
validation_pred_rfc2 = np.zeros(len(xtrain))
scores_rfc2 = []
valid_rfc2 = np.zeros(len(label))
for fold_,(xtrn_indx, xvad_indx)in enumerate(folds.split(xtrain, label)) :
print('Fold {}, started at {}'.format(fold_, time.ctime()))
x_train, x_valid = xtrain[xtrn_indx], xtrain[xvad_indx]
y_train, y_valid = label[xtrn_indx], label[xvad_indx]
clf_rfc2 = RandomForestClassifier(n_estimators=1000, n_jobs=-1, random_state=0)
 clf_rfc2.fit(x_train, y_train)
 pred_valid = clf_rfc2.predict(x_valid ).reshape(-1,)
 score = accuracy_score(y_valid, pred_valid)
 Pred_Real = clf_rfc2.predict_proba(xtest)[:, 1]
validation_pred_rfc2[xvad_indx] +=(pred_valid ).reshape(-1,)
 scores_rfc2.append(accuracy_score(y_valid, pred_valid))
forcast_rfc2 += Pred_Real
valid_rfc2[xvad_indx] +=(y_valid)
 print(c_report(valid_rfc2[xvad_indx], validation_pred_rfc2[xvad_indx]))
print(c_matrix(valid_rfc2[xvad_indx], validation_pred_rfc2[xvad_indx]))
print('accuracy is: {}, std: {}.'.format(np.mean(scores_rfc2), np.std(scores_rfc2)) )<find_best_model_class>
netGE = Generator(**config['Generator'] ).to(device, torch.float32)
 netGE.load_state_dict(netG.state_dict());
forcast_dtc2 = np.zeros(len(xtest))
validation_pred_dtc2 = np.zeros(len(xtrain))
scores_dtc2 = []
valid_dtc2 = np.zeros(len(label))
for fold_,(xtrn_indx, xvad_indx)in enumerate(folds.split(xtrain, label)) :
print('Fold {}, started at {}'.format(fold_, time.ctime()))
x_train, x_valid = xtrain[xtrn_indx], xtrain[xvad_indx]
y_train, y_valid = label[xtrn_indx], label[xvad_indx]
clf_dtc2 = DecisionTreeClassifier()
clf_dtc2.fit(x_train, y_train)
 pred_valid = clf_dtc2.predict(x_valid ).reshape(-1,)
 score = accuracy_score(y_valid, pred_valid)
 Pred_Real = clf_dtc2.predict_proba(xtest)[:, 1]
validation_pred_dtc2[xvad_indx] +=(pred_valid ).reshape(-1,)
 scores_dtc2.append(accuracy_score(y_valid, pred_valid))
forcast_dtc2 += Pred_Real
valid_dtc2[xvad_indx] +=(y_valid)
 print(c_report(valid_dtc2[xvad_indx], validation_pred_dtc2[xvad_indx]))
print(c_matrix(valid_dtc2[xvad_indx], validation_pred_dtc2[xvad_indx]))
print('accuracy is: {}, std: {}.'.format(np.mean(scores_dtc2), np.std(scores_dtc2)) )<train_model>
optim_G = Adam(params=netG.parameters() , lr=config['lr_G'], betas=config['betas'])
 optim_D = Adam(params=netD.parameters() , lr=config['lr_D'], betas=config['betas'] )
forcast_svm2 = np.zeros(len(xtest))
validation_pred_svm2 = np.zeros(len(xtrain))
scores_svm2 = []
valid_svm2 = np.zeros(len(label))
for fold_,(xtrn_indx, xvad_indx)in enumerate(folds.split(xtrain, label)) :
print('Fold {}, started at {}'.format(fold_, time.ctime()))
x_train, x_valid = xtrain[xtrn_indx], xtrain[xvad_indx]
y_train, y_valid = label[xtrn_indx], label[xvad_indx]
clf_svm2 = svm.SVC(kernel='linear', gamma=1)
 clf_svm2.fit(x_train, y_train)
 pred_valid = clf_svm2.predict(x_valid ).reshape(-1,)
 score = accuracy_score(y_valid, pred_valid)
 Pred_Real = clf_svm2.predict(xtest)
 validation_pred_svm2[xvad_indx] +=(pred_valid ).reshape(-1,)
 scores_svm2.append(accuracy_score(y_valid, pred_valid))
forcast_svm2 += Pred_Real
valid_svm2[xvad_indx] +=(y_valid)
 print(c_report(valid_svm2[xvad_indx], validation_pred_svm2[xvad_indx]))
print(c_matrix(valid_svm2[xvad_indx], validation_pred_svm2[xvad_indx]))
print('accuracy is: {}, std: {}.'.format(np.mean(scores_svm2), np.std(scores_svm2)) )<train_model>
decay_iter = config['num_iterations'] - config['decay_start_iteration']
if decay_iter > 0:
lr_lambda_G = lambda x:(max(0,1-x/decay_iter))
lr_lambda_D = lambda x:(max(0,1-x/(decay_iter*config['d_steps'])))
 lr_sche_G = LambdaLR(optim_G, lr_lambda=lr_lambda_G)
 lr_sche_D = LambdaLR(optim_D, lr_lambda=lr_lambda_D )
forcast_lr2 = np.zeros(len(xtest))
validation_pred_lr2 = np.zeros(len(xtrain))
scores_lr2 = []
valid_lr2 = np.zeros(len(label))
for fold_,(xtrn_indx, xvad_indx)in enumerate(folds.split(xtrain, label)) :
print('Fold {}, started at {}'.format(fold_, time.ctime()))
x_train, x_valid = xtrain[xtrn_indx], xtrain[xvad_indx]
y_train, y_valid = label[xtrn_indx], label[xvad_indx]
clf_lr2 = linear_model.LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear')
 clf_lr2.fit(x_train, y_train)
 pred_valid = clf_lr2.predict(x_valid ).reshape(-1,)
 score = accuracy_score(y_valid, pred_valid)
 Pred_Real = clf_lr2.predict_proba(xtest)[:, 1]
validation_pred_lr2[xvad_indx] +=(pred_valid ).reshape(-1,)
 scores_lr2.append(accuracy_score(y_valid, pred_valid))
forcast_lr2 += Pred_Real
valid_lr2[xvad_indx] +=(y_valid)
 print(c_report(valid_lr2[xvad_indx], validation_pred_lr2[xvad_indx]))
print(c_matrix(valid_lr2[xvad_indx], validation_pred_lr2[xvad_indx]))
print('accuracy is: {}, std: {}.'.format(np.mean(scores_lr2), np.std(scores_lr2)) )<train_model>
def calc_advloss_D(real, fake, margin=1.0):
loss_real = torch.mean(( real - fake.mean() - margin)** 2)
 loss_fake = torch.mean(( fake - real.mean() + margin)** 2)
 loss =(loss_real + loss_fake)/ 2
return loss
perm = PermutationImportance(clf_lr2, random_state=1 ).fit(xtrain, label)
 eli5.show_weights(perm, top=50 )<normalization>
def calc_advloss_G(real, fake, margin=1.0):
loss_real = torch.mean(( real - fake.mean() + margin)** 2)
 loss_fake = torch.mean(( fake - real.mean() - margin)** 2)
 loss =(loss_real + loss_fake)/ 2
return loss
top_features = [i[1:] for i in eli5.formatters.as_dataframe.explain_weights_df(clf_lr2 ).feature if 'BIAS' not in i]
xtrain_lr_elif = train[top_features]
xtest_lr_elif = test[top_features]
scaler = StandardScaler()
xtrain_lr_elif = scaler.fit_transform(xtrain_lr_elif)
 xtest_lr_elif = scaler.transform(xtest_lr_elif)
 forcast_lr4 = np.zeros(len(xtest_lr_elif))
validation_pred_lr4 = np.zeros(len(xtrain_lr_elif))
scores_lr4 = []
valid_lr4 = np.zeros(len(label))
for fold_,(xtrn_indx, xvad_indx)in enumerate(folds.split(xtrain_lr_elif, label)) :
print('Fold {}, started at {}'.format(fold_, time.ctime()))
x_train, x_valid = xtrain_lr_elif[xtrn_indx], xtrain_lr_elif[xvad_indx]
y_train, y_valid = label[xtrn_indx], label[xvad_indx]
clf_lr4 = linear_model.LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear')
 clf_lr4.fit(x_train, y_train)
 pred_valid = clf_lr4.predict(x_valid ).reshape(-1,)
 score = accuracy_score(y_valid, pred_valid)
 Pred_Real = clf_lr4.predict_proba(xtest_lr_elif)[:, 1]
validation_pred_lr4[xvad_indx] +=(pred_valid ).reshape(-1,)
 scores_lr4.append(accuracy_score(y_valid, pred_valid))
forcast_lr4 += Pred_Real
valid_lr4[xvad_indx] +=(y_valid)
 print(c_report(valid_lr4[xvad_indx], validation_pred_lr4[xvad_indx]))
print(c_matrix(valid_lr4[xvad_indx], validation_pred_lr4[xvad_indx]))
print('accuracy is: {}, std: {}.'.format(np.mean(scores_lr4), np.std(scores_lr4)) )<train_model>
criterion = nn.NLLLoss().to(device, torch.float32 )
perm = PermutationImportance(clf_lr2, random_state=1 ).fit(xtrain, label)
 eli5.show_weights(perm, top=50 )<normalization>
def sample_latents(batch_size, latent_dim, num_classes):
latents = torch.randn(( batch_size, latent_dim), dtype=torch.float32, device=device)
 labels = torch.randint(0, num_classes, size=(batch_size,), dtype=torch.long, device=device)
 return latents, labels
top_features = [i[1:] for i in eli5.formatters.as_dataframe.explain_weights_df(perm ).feature if 'BIAS' not in i]
xtrain_lr_perm = train[top_features]
xtest_lr_perm = test[top_features]
scaler = StandardScaler()
xtrain_lr_perm = scaler.fit_transform(xtrain_lr_perm)
 xtest_lr_perm = scaler.transform(xtest_lr_perm)
 forcast_lr3 = np.zeros(len(xtest_lr_perm))
validation_pred_lr3 = np.zeros(len(xtrain_lr_perm))
scores_lr3 = []
valid_lr3 = np.zeros(len(label))
for fold_,(xtrn_indx, xvad_indx)in enumerate(folds.split(xtrain_lr_perm, label)) :
print('Fold {}, started at {}'.format(fold_, time.ctime()))
x_train, x_valid = xtrain_lr_perm[xtrn_indx], xtrain_lr_perm[xvad_indx]
y_train, y_valid = label[xtrn_indx], label[xvad_indx]
clf_lr3 = linear_model.LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear')
 clf_lr3.fit(x_train, y_train)
 pred_valid = clf_lr3.predict(x_valid ).reshape(-1,)
 score = accuracy_score(y_valid, pred_valid)
 Pred_Real = clf_lr3.predict_proba(xtest_lr_perm)[:, 1]
validation_pred_lr3[xvad_indx] +=(pred_valid ).reshape(-1,)
 scores_lr3.append(accuracy_score(y_valid, pred_valid))
forcast_lr3 += Pred_Real
valid_lr3[xvad_indx] +=(y_valid)
 print(c_report(valid_lr3[xvad_indx], validation_pred_lr3[xvad_indx]))
print(c_matrix(valid_lr3[xvad_indx], validation_pred_lr3[xvad_indx]))
print('accuracy is: {}, std: {}.'.format(np.mean(scores_lr3), np.std(scores_lr3)) )<train_model>
step = 1
while True:
for i in range(config['d_steps']):
for param in netD.parameters() :
param.requires_grad_(True)
 optim_D.zero_grad()
real_imgs, real_labels = train_dataiterator.__next__()
batch_size = real_imgs.size(0)
 latents, fake_labels = sample_latents(batch_size, **config['sample_latents'])
 fake_imgs = netG(latents, fake_labels ).detach()
preds_real, preds_real_labels = netD(real_imgs, real_labels)
 preds_fake, _ = netD(fake_imgs, fake_labels)
 loss_D = calc_advloss_D(preds_real, preds_fake, config['margin'])
 loss_D += config['gamma'] * criterion(preds_real_labels, real_labels)
 loss_D.backward()
optim_D.step()
if(decay_iter > 0)and(step > config['decay_start_iteration']):
lr_sche_D.step()
for param in netD.parameters() :
param.requires_grad_(False)
 optim_G.zero_grad()
real_imgs, real_labels = train_dataiterator.__next__()
batch_size = real_imgs.size(0)
 latents, fake_labels = sample_latents(batch_size, **config['sample_latents'])
 fake_imgs = netG(latents, fake_labels)
 preds_real, _ = netD(real_imgs, real_labels)
 preds_fake, preds_fake_labels = netD(fake_imgs, fake_labels)
 loss_G = calc_advloss_G(preds_real, preds_fake, config['margin'])
 loss_G += config['gamma'] * criterion(preds_fake_labels, fake_labels)
 loss_G.backward()
optim_G.step()
if(decay_iter > 0)and(step > config['decay_start_iteration']):
lr_sche_G.step()
for param_G, param_GE in zip(netG.parameters() , netGE.parameters()):
param_GE.data.mul_(config['ema'] ).add_(( 1-config['ema'])*param_G.data)
 for buffer_G, buffer_GE in zip(netG.buffers() , netGE.buffers()):
buffer_GE.data.mul_(config['ema'] ).add_(( 1-config['ema'])*buffer_G.data)
 if step < config['num_iterations']:
step += 1
else:
print('total step: {}'.format(step))
break
forcast_bc = np.zeros(len(xtest))
validation_pred_bc = np.zeros(len(xtrain))
scores_bc = []
valid_bc = np.zeros(len(label))
for fold_,(xtrn_indx, xvad_indx)in enumerate(folds.split(xtrain, label)) :
print('Fold {}, started at {}'.format(fold_, time.ctime()))
x_train, x_valid = xtrain[xtrn_indx], xtrain[xvad_indx]
y_train, y_valid = label[xtrn_indx], label[xvad_indx]
clf_bc = BaggingClassifier()
clf_bc.fit(x_train, y_train)
 pred_valid = clf_bc.predict(x_valid ).reshape(-1,)
 score = accuracy_score(y_valid, pred_valid)
 Pred_Real = clf_bc.predict_proba(xtest)[:, 1]
validation_pred_bc[xvad_indx] +=(pred_valid ).reshape(-1,)
 scores_bc.append(accuracy_score(y_valid, pred_valid))
forcast_bc += Pred_Real
valid_bc[xvad_indx] +=(y_valid)
 print(c_report(valid_bc[xvad_indx], validation_pred_bc[xvad_indx]))
print(c_matrix(valid_bc[xvad_indx], validation_pred_bc[xvad_indx]))
print('accuracy is: {}, std: {}.'.format(np.mean(scores_bc), np.std(scores_bc)) )<choose_model_class>
def truncated_normal(size, threshold=2.0, dtype=torch.float32, device='cpu'):
x = scipy.stats.truncnorm.rvs(-threshold, threshold, size=size)
 x = torch.from_numpy(x ).to(device, dtype)
 return x
<choose_model_class><EOS> 
def generate_eval_samples(generator, batch_size, latent_dim, num_classes):
latents = truncated_normal(( batch_size, latent_dim), dtype=torch.float32, device=device)
 labels = torch.randint(0, num_classes, size=(batch_size,), dtype=torch.long, device=device)
 with torch.no_grad() :
imgs =(generator(latents, labels)+ 1)/ 2
return imgs