Kaggle competition: ALASKA2 Image Steganalysis
<SOS> metric: WeightedAUC Kaggle data source: alaska2-image-steganalysis<feature_engineering>
!pip install stegano
!pip install -q efficientnet_pytorch
new_data['original_and_new']=new_data['original_and_new'].apply(lambda x: 1 if x else 0 )<define_variables>
%matplotlib inline
RandomBrightness, RandomContrast, HueSaturationValue, Blur, GaussNoise)
 warnings.filterwarnings("ignore" )
list_geres_name=['Comedy','Thriller','Action','Drama','Romance']<feature_engineering>
original_vector = np.array(original ).flatten()
altered_vector = np.array(altered ).flatten()
print('Original shape:', original_vector.shape, '
' +
'Altered shape:', altered_vector.shape)
 dist1 = np.sum(original_vector - original_vector)
 print('Dist1:', dist1)
 dist2 = np.sum(original_vector - altered_vector)
 print('Dist2:', dist2 )
for i in list_geres_name:
new_data['geres_name'+'_'+i]=new_data['geres_name'].apply(lambda x: 1 if i in x else 0 )<drop_column>
base_path = '.. /input/alaska2-image-steganalysis'
def read_images_path(dir_name='Cover', test = False):

 series_name = pd.Series(os.listdir(base_path + '/' + dir_name))
if test:
series_name = pd.Series(os.listdir(base_path + '/' + 'Test'))
series_paths = pd.Series(base_path + '/' + dir_name + '/' + series_name)
 return series_paths
new_data=new_data.drop('geres_name',axis=1 )<data_type_conversions>
cover_paths = read_images_path('Cover', False)
 jmipod_paths = read_images_path('JMiPOD', False)
 juniward_paths = read_images_path('JUNIWARD', False)
 uerd_paths = read_images_path('UERD', False)
 test_paths = read_images_path('Test', True )
new_data['production_countries']=new_data['production_countries'].fillna('QQ' )<feature_engineering>
image_sample = mpimg.imread(cover_paths[0])
 print('Image sample shape:', image_sample.shape)
 print('Image sample size:', image_sample.size)
 print('Image sample data type:', image_sample.dtype )
new_data['production_countries'].loc[new_data['production_countries']=='Unknow']['production_countries']='QQ'<count_unique_values>
def dct2(a):
return fftpack.dct(fftpack.dct(a, axis=0, norm='ortho'), axis=1, norm='ortho')
 imsize = image.shape
dct = np.zeros(imsize)
 for i in r_[:imsize[0]:8]:
for j in r_[:imsize[1]:8]:
dct[i:(i+8),j:(j+8)] = dct2(image[i:(i+8),j:(j+8)] )
list_pro_coun=list(new_data['production_countries'])
 d = Counter([j for i in list_pro_coun for j in i] ).most_common(5)
 d<feature_engineering>
def set_seed(seed = 1234):

 np.random.seed(seed)
 random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
os.environ['PYTHONHASHSEED'] = str(seed)
 set_seed()
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 print('Device available now:', device )
for i in d:
new_data['production_cou_name'+'_'+i[0]]=new_data['production_countries'].apply(lambda x: 1 if i[0] in x else 0 )<feature_engineering>
sample_size = 256
num_classes = 4
train_df = pd.read_csv('.. /input/alaska2-trainvalid-4-class-csv/alaska2_train_data_4classes.csv',
header=0, names=['Path', 'Label'], dtype = {'Label':np.int32})
 valid_df = pd.read_csv('.. /input/alaska2-trainvalid-4-class-csv/alaska2_valid_data_4classes.csv',
header=0, names=['Path', 'Label'], dtype = {'Label':np.int32})
 def sample_data(dataframe, sample_size, num_classes, train=True):

 if train:
size = int(0.75 * sample_size)
 else:
size = int(0.25 * sample_size)
 no = int(np.floor(size/num_classes))
labels = [i for i in range(num_classes)]
new_data = pd.DataFrame()
for label in labels:
data = dataframe[dataframe['Label'] == label].sample(no, random_state=123)
 new_data = pd.concat([new_data, data], axis=0)
 return new_data
new_data['pro_country_count']=new_data['production_companies'].apply(lambda x:len(x))<groupby>
class AlaskaDataset(Dataset):

 def __init__(self, dataframe, is_test, is_val, vertical_flip=0.5, horizontal_flip=0.5):
self.dataframe, self.is_test, self.is_val = dataframe, is_test, is_val
self.vertical_flip, self.horizontal_flip = vertical_flip, horizontal_flip
flag = is_test or is_val
if flag:
self.transform = Compose([Resize(512, 512),
Normalize() ,
ToFloat(max_value=255),
ToTensor() ])
 else:
self.transform = Compose([Resize(512, 512),
VerticalFlip(p = vertical_flip),
HorizontalFlip(p = horizontal_flip),
Normalize() ,
ToFloat(max_value=255),
ToTensor() ])
 def __len__(self):
return len(self.dataframe)
 def __getitem__(self, index):
if self.is_test:
path = self.dataframe.loc[index][0]
else:
path, label = self.dataframe.loc[index]
image = cv2.imread(path)[:, :, ::-1]
image = self.transform(image=image)
 image = image['image']
if self.is_test:
return image
else:
return image, label
pro_count_rev=new_data.loc[train.index].groupby('pro_country_count' ).revenue.median()<drop_column>
class EfficientNetwork(nn.Module):
def __init__(self, output_size, b1=False, b2=False):
super().__init__()
self.b1, self.b2 = b1, b2
if b1:
self.features = EfficientNet.from_pretrained('efficientnet-b1')
 elif b2:
self.features = EfficientNet.from_pretrained('efficientnet-b2')
 else:
self.features = EfficientNet.from_pretrained('efficientnet-b0')
 if b1:
self.classification = nn.Linear(1280, output_size)
 elif b2:
self.classification = nn.Linear(1408, output_size)
 else:
self.classification = nn.Linear(1280, output_size)
 def forward(self, image, prints=False):
if prints: print('Input Image shape:', image.shape)
 image = self.features.extract_features(image)
 if prints: print('Features Image shape:', image.shape)
 if self.b1:
image = F.avg_pool2d(image, image.size() [2:] ).reshape(-1, 1280)
 elif self.b2:
image = F.avg_pool2d(image, image.size() [2:] ).reshape(-1, 1408)
 else:
image = F.avg_pool2d(image, image.size() [2:] ).reshape(-1, 1280)
 if prints: print('Image Reshaped shape:', image.shape)
 out = self.classification(image)
 if prints: print('Out shape:', out.shape)
 return out
new_data=new_data.drop('production_countries',axis=1 )<categorify>
model_example = EfficientNetwork(output_size=num_classes, b1=False, b2=True )
new_data['production_companies'].loc[new_data['production_companies']=='unknow']='u'<drop_column>
example_data = AlaskaDataset(train_df, is_test=False, is_val=False)
 example_loader = torch.utils.data.DataLoader(example_data, batch_size = 1, shuffle=True)
 for image, labels in example_loader:
images_example = image
labels_example = torch.tensor(labels, dtype=torch.long)
 break
print('Images shape:', images_example.shape)
 print('Labels:', labels, '
')
 out = model_example(images_example, prints=True)
 criterion_example = nn.CrossEntropyLoss()
loss = criterion_example(out, labels_example)
 print('Loss:', loss.item() )
d.remove(( 'u',414)) 
<concatenate>
class ResNet34Network(nn.Module):
def __init__(self, output_size):
super().__init__()
self.features = resnet34(pretrained=True)
 self.classification = nn.Linear(1000, output_size)
 def forward(self, image, prints=False):
if prints: print('Input Image shape:', image.shape)
 image = self.features(image)
 if prints: print('Features Image shape:', image.shape)
 out = self.classification(image)
 if prints: print('Out shape:', out.shape)
 return out
dd=[]
for i in d:
dd.append(i[0] )<concatenate>
model_example = ResNet34Network(output_size=num_classes )
dd=[]
for i in d:
dd.append(i[0] )<drop_column>
example_data = AlaskaDataset(train_df, is_test=False, is_val=False)
 example_loader = torch.utils.data.DataLoader(example_data, batch_size = 1, shuffle=True)
 for image, labels in example_loader:
images_example = image
labels_example = torch.tensor(labels, dtype=torch.long)
 break
print('Images shape:', images_example.shape)
 print('Labels:', labels, '
')
 out = model_example(images_example, prints=True)
 criterion_example = nn.CrossEntropyLoss()
loss = criterion_example(out, labels_example)
 print('Loss:', loss.item() )
new_data=new_data.drop('spoken_languages',axis=1 )<feature_engineering>
vertical_flip = 0.5
horizontal_flip = 0.5
new_data['runtime'].loc[new_data['runtime']==0]=1
new_data['time_budget']=new_data['budget'].fillna(0)/new_data['runtime'].fillna(1 )<drop_column>
train_data = AlaskaDataset(train_df, is_test=False, is_val=False,
vertical_flip=vertical_flip, horizontal_flip=horizontal_flip)
 valid_data = AlaskaDataset(valid_df, is_test=False, is_val=True,
vertical_flip=vertical_flip, horizontal_flip=horizontal_flip )
new_data=new_data.drop('production_companies',axis=1 )<drop_column>
def alaska_weighted_auc(y_true, y_valid):
tpr_thresholds = [0.0, 0.4, 1.0]
weights = [2, 1]
fpr, tpr, thresholds = metrics.roc_curve(y_true, y_valid, pos_label=1)
 areas = np.array(tpr_thresholds[1:])- np.array(tpr_thresholds[:-1])
 normalization = np.dot(areas, weights)
 competition_metric = 0
for idx, weight in enumerate(weights):
y_min = tpr_thresholds[idx]
y_max = tpr_thresholds[idx + 1]
mask =(y_min < tpr)&(tpr < y_max)
 x_padding = np.linspace(fpr[mask][-1], 1, 100)
 x = np.concatenate([fpr[mask], x_padding])
 y = np.concatenate([tpr[mask], [y_max] * len(x_padding)])
 y = y - y_min
score = metrics.auc(x, y)
 submetric = score * weight
best_subscore =(y_max - y_min)* weight
competition_metric += submetric
return competition_metric / normalization
new_data=new_data.drop('crew',axis=1 )<feature_engineering>
def train(model, epochs, batch_size, num_workers, learning_rate, weight_decay,
version = 'vx', plot_loss=False):
f = open(f"logs_{version}.txt", "w+")
 best_auc = None
train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size, num_workers=num_workers,
drop_last=True, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid_data, batch_size=batch_size, num_workers=num_workers,
drop_last=True, shuffle=True)
 criterion = torch.nn.CrossEntropyLoss()
optimizer = optim.AdamW(model.parameters() , lr=learning_rate, weight_decay=weight_decay)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer=optimizer, mode='max',
patience=1, verbose=True, factor=0.4)
 train_losses = []
evaluation_losses = []
for epoch in range(epochs):
model.train()
train_loss = 0
for images, labels in train_loader:
images = images.to(device, dtype=torch.float)
 labels = labels.to(device, dtype=torch.long)
 optimizer.zero_grad()
out = model(images)
 loss = criterion(out, labels)
 loss.backward()
optimizer.step()
train_loss += loss.item()
epoch_loss_train = train_loss / batch_size
train_losses.append(epoch_loss_train)
 model.eval()
evaluation_loss = 0
actuals, predictions = [], []
with torch.no_grad() :
for images, labels in valid_loader:
images = images.to(device, dtype=torch.float)
 labels = labels.to(device, dtype=torch.long)
 out = model(images)
 loss = criterion(out, labels)
 actuals.extend(labels.cpu().numpy().astype(int))
predictions.extend(F.softmax(out, 1 ).cpu().numpy())
 evaluation_loss += loss.item()
epoch_loss_eval = evaluation_loss/batch_size
evaluation_losses.append(epoch_loss_eval)
 predictions = np.array(predictions)
 predicted_labels = predictions.argmax(1)
 accuracy =(predicted_labels == actuals ).mean()
new_preds = np.zeros(len(predictions))
temp = predictions[predicted_labels != 0, 1:]
new_preds[predicted_labels != 0] = temp.sum(1)
 new_preds[predicted_labels == 0] = 1 - predictions[predicted_labels == 0, 0]
actuals = np.array(actuals)
 actuals[actuals != 0] = 1
auc_score = alaska_weighted_auc(actuals, new_preds)
 with open(f"logs_{version}.txt", 'a+')as f:
print('Epoch: {}/{} | Train Loss: {:.3f} | Eval Loss: {:.3f} | AUC: {:3f} | Acc: {:3f}'.\
format(epoch+1, epochs, epoch_loss_train, epoch_loss_eval, auc_score, accuracy), file=f)
 print('Epoch: {}/{} | Train Loss: {:.3f} | Eval Loss: {:.3f} | AUC: {:3f} | Acc: {:3f}'.\
format(epoch+1, epochs, epoch_loss_train, epoch_loss_eval, auc_score, accuracy))
if best_auc == None:
best_auc = auc_score
torch.save(model.state_dict() ,
f"Epoch_{epoch+1}_ValLoss_{epoch_loss_eval:.3f}_AUC_{auc_score:.3f}.pth")
 continue
if auc_score > best_auc:
best_auc = auc_score
torch.save(model.state_dict() ,
f"Epoch_{epoch+1}_ValLoss_{epoch_loss_eval:.3f}_AUC_{auc_score:.3f}.pth")
 scheduler.step(auc_score)
 if plot_loss:
plt.figure(figsize=(14,5))
plt.plot(train_losses, c='
plt.plot(evaluation_losses, c='
plt.legend(['Train Loss', 'Evaluation Loss'])
 plt.title('Losses over Epochs');
for i in list1:
new_data['title_is'+i]=new_data['title'].fillna('' ).apply(lambda x: 1 if i in x else 0 )<feature_engineering>
version = 'v8'
epochs = 10
batch_size = 32
num_workers = 8
learning_rate = 0.0001
weight_decay = 0.00001
plot_loss = False
for col in ['tagline', 'overview','title']:
new_data['len_' + col] =new_data[col].fillna('' ).apply(lambda x: len(x))
new_data['words_' + col] = new_data[col].fillna('' ).apply(lambda x: len(x.split(' ')))
 new_data=new_data.drop(col,axis=1 )<feature_engineering>
eff_net2 = EfficientNetwork(output_size = num_classes, b1=False, b2=True ).to(device)
new_data['budget_popularity']=new_data['budget']*1.0/new_data['popularity']<count_values>
directory = '.. /input/alaska2-image-steganalysis/'
name = pd.Series(sorted(os.listdir(directory + 'Test/')))
 path = pd.Series(directory + 'Test/' + name)
 test_df = pd.DataFrame(data=path, columns=['Path'])
 test_data = AlaskaDataset(test_df, is_test=True, is_val=False,
vertical_flip=vertical_flip, horizontal_flip=horizontal_flip)
 test_loader = torch.utils.data.DataLoader(test_data, batch_size = batch_size, shuffle=False )
a=new_data.groupby(['release_year'] ).release_month.value_counts()
b=new_data.groupby(['release_year','release_month'] ).release_day.value_counts()
new_data[(new_data['release_year']==2017)&(new_data['release_day']==30)] 
<feature_engineering>
eff_net2.eval()
predictions = []
with torch.no_grad() :
for k, images in enumerate(test_loader):
images = images.to(device)
 out0 = eff_net2(images)
 images_vertical = images.flip(2)
 out1 = eff_net2(images_vertical)
 images_flip = images.flip(3)
 out2 = eff_net2(images_flip)
 outputs =(0.25*out1 + 0.25*out2)
 outputs =(outputs + 0.5*out0)
 predictions.extend(F.softmax(outputs, 1 ).cpu().numpy() )
new_data['_releaseYear_popularity_ratio'] = new_data['release_year'] / new_data['popularity']
new_data['runtime_to_mean_year'] = new_data['runtime'] / new_data.groupby("release_year")["runtime"].transform('mean')
 new_data['popularity_to_mean_year'] = new_data['popularity'] / new_data.groupby("release_year")["popularity"].transform('mean')
 new_data['budget_to_mean_year'] = new_data['budget'] / new_data.groupby("release_year")["budget"].transform('mean')
<feature_engineering>
predictions = np.array(predictions)
 predicted_labels = predictions.argmax(1)
 new_preds = np.zeros(len(predictions))
temp = predictions[predicted_labels != 0, 1:]
new_preds[predicted_labels != 0] = temp.sum(1)
 new_preds[predicted_labels == 0] = 1 - predictions[predicted_labels == 0, 0]
<feature_engineering>
ss = pd.read_csv('.. /input/alaska2-image-steganalysis/sample_submission.csv')
 ss['Label'] = new_preds
ss.to_csv(f'submission_{version}.csv', index=False )
<split>
!pip install -q efficientnet_pytorch > /dev/null
new_train=new_data.loc[train.index]
new_test=new_data.loc[test.index]<train_model>
SEED = 42
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(SEED )
x=new_train.drop('revenue',axis=1)
 y=new_train['revenue']
x=x.fillna(0)
 decision_tree = DecisionTreeRegressor(max_depth = 6)
 decision_tree.fit(x, y)
<train_model>
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=42)
 model_XG = xgboost.XGBRegressor()
model_XG.fit(x_train,y_train)
 y_predict_rf = model_XG.predict(x_test)
 print(mean_squared_error(y_test, y_predict_rf)) 
<drop_column>
def get_train_transforms() :
return A.Compose([
A.HorizontalFlip(p=0.5),
A.VerticalFlip(p=0.5),
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
], p=1.0)
 def get_valid_transforms() :
return A.Compose([
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )
X_test=new_test.drop('revenue',axis=1)
 X_test=X_test.fillna(0 )<predict_on_test>
DATA_ROOT_PATH = '.. /input/alaska2-image-steganalysis'
def onehot(size, target):
vec = torch.zeros(size, dtype=torch.float32)
 vec[target] = 1.
 return vec
class DatasetRetriever(Dataset):
def __init__(self, kinds, image_names, labels, transforms=None):
super().__init__()
self.kinds = kinds
self.image_names = image_names
self.labels = labels
self.transforms = transforms
def __getitem__(self, index: int):
kind, image_name, label = self.kinds[index], self.image_names[index], self.labels[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{kind}/{image_name}', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
target = onehot(4, label)
 return image, target
def __len__(self)-> int:
return self.image_names.shape[0]
def get_labels(self):
return list(self.labels )
ypred=model_XG.predict(X_test )<prepare_x_and_y>
fold_number = 0
train_dataset = DatasetRetriever(
 kinds=dataset[dataset['fold'] != fold_number].kind.values,
image_names=dataset[dataset['fold'] != fold_number].image_name.values,
labels=dataset[dataset['fold'] != fold_number].label.values,
transforms=get_train_transforms() , 
)
 validation_dataset = DatasetRetriever(
 kinds=dataset[dataset['fold'] == fold_number].kind.values,
image_names=dataset[dataset['fold'] == fold_number].image_name.values,
labels=dataset[dataset['fold'] == fold_number].label.values,
transforms=get_valid_transforms() ,
)
dtrain = xgb.DMatrix(x, y)
 dtest = xgb.DMatrix(X_test )<train_on_grid>
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
def alaska_weighted_auc(y_true, y_valid):

 tpr_thresholds = [0.0, 0.4, 1.0]
weights = [2, 1]
fpr, tpr, thresholds = metrics.roc_curve(y_true, y_valid, pos_label=1)
 areas = np.array(tpr_thresholds[1:])- np.array(tpr_thresholds[:-1])
 normalization = np.dot(areas, weights)
 competition_metric = 0
for idx, weight in enumerate(weights):
y_min = tpr_thresholds[idx]
y_max = tpr_thresholds[idx + 1]
mask =(y_min < tpr)&(tpr < y_max)
 x_padding = np.linspace(fpr[mask][-1], 1, 100)
 x = np.concatenate([fpr[mask], x_padding])
 y = np.concatenate([tpr[mask], [y_max] * len(x_padding)])
 y = y - y_min
score = metrics.auc(x, y)
 submetric = score * weight
best_subscore =(y_max - y_min)* weight
competition_metric += submetric
return competition_metric / normalization
class RocAucMeter(object):
def __init__(self):
self.reset()
def reset(self):
self.y_true = np.array([0,1])
 self.y_pred = np.array([0.5,0.5])
 self.score = 0
def update(self, y_true, y_pred):
y_true = y_true.cpu().numpy().argmax(axis=1 ).clip(min=0, max=1 ).astype(int)
 y_pred = 1 - nn.functional.softmax(y_pred, dim=1 ).data.cpu().numpy() [:,0]
self.y_true = np.hstack(( self.y_true, y_true))
self.y_pred = np.hstack(( self.y_pred, y_pred))
self.score = alaska_weighted_auc(self.y_true, self.y_pred)
 @property
def avg(self):
return self.score
xgb_params = {
'eta': 0.08,
'max_depth': 15,
'subsample': 0.7,
'colsample_bytree': 0.7,
'objective': 'reg:linear',
'eval_metric': 'rmse',
'silent': 1
}
cv_output = xgb.cv(xgb_params, dtrain, num_boost_round=1000, early_stopping_rounds=400, verbose_eval=50, show_stdv=False)
 cv_output[['train-rmse-mean', 'test-rmse-mean']].plot()<predict_on_test>
class LabelSmoothing(nn.Module):
def __init__(self, smoothing = 0.05):
super(LabelSmoothing, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
def forward(self, x, target):
if self.training:
x = x.float()
target = target.float()
logprobs = torch.nn.functional.log_softmax(x, dim = -1)
 nll_loss = -logprobs * target
nll_loss = nll_loss.sum(-1)
 smooth_loss = -logprobs.mean(dim=-1)
 loss = self.confidence * nll_loss + self.smoothing * smooth_loss
return loss.mean()
else:
return torch.nn.functional.cross_entropy(x, target )
y_predict1 = abs(model.predict(dtest))<prepare_x_and_y>
warnings.filterwarnings("ignore")
 class Fitter:
def __init__(self, model, device, config):
self.config = config
self.epoch = 0
self.base_dir = './'
self.log_path = f'{self.base_dir}/log.txt'
self.best_summary_loss = 10**5
self.model = model
self.device = device
param_optimizer = list(self.model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
self.optimizer = torch.optim.AdamW(self.model.parameters() , lr=config.lr)
 self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)
 self.criterion = LabelSmoothing().to(self.device)
 self.log(f'Fitter prepared.Device is {self.device}')
 def fit(self, train_loader, validation_loader):
for e in range(self.config.n_epochs):
if self.config.verbose:
lr = self.optimizer.param_groups[0]['lr']
timestamp = datetime.utcnow().isoformat()
self.log(f'
{timestamp}
LR: {lr}')
 t = time.time()
summary_loss, final_scores = self.train_one_epoch(train_loader)
 self.log(f'[RESULT]: Train.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, final_score: {final_scores.avg:.5f}, time: {(time.time() - t):.5f}')
 self.save(f'{self.base_dir}/last-checkpoint.bin')
 t = time.time()
summary_loss, final_scores = self.validation(validation_loader)
 self.log(f'[RESULT]: Val.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, final_score: {final_scores.avg:.5f}, time: {(time.time() - t):.5f}')
 if summary_loss.avg < self.best_summary_loss:
self.best_summary_loss = summary_loss.avg
self.model.eval()
self.save(f'{self.base_dir}/best-checkpoint-{str(self.epoch ).zfill(3)}epoch.bin')
 for path in sorted(glob(f'{self.base_dir}/best-checkpoint-*epoch.bin')) [:-3]:
os.remove(path)
 if self.config.validation_scheduler:
self.scheduler.step(metrics=summary_loss.avg)
 self.epoch += 1
def validation(self, val_loader):
self.model.eval()
summary_loss = AverageMeter()
final_scores = RocAucMeter()
t = time.time()
for step,(images, targets)in enumerate(val_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Val Step {step}/{len(val_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, final_score: {final_scores.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 with torch.no_grad() :
targets = targets.to(self.device ).float()
batch_size = images.shape[0]
images = images.to(self.device ).float()
outputs = self.model(images)
 loss = self.criterion(outputs, targets)
 final_scores.update(targets, outputs)
 summary_loss.update(loss.detach().item() , batch_size)
 return summary_loss, final_scores
def train_one_epoch(self, train_loader):
self.model.train()
summary_loss = AverageMeter()
final_scores = RocAucMeter()
t = time.time()
for step,(images, targets)in enumerate(train_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Train Step {step}/{len(train_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, final_score: {final_scores.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 targets = targets.to(self.device ).float()
images = images.to(self.device ).float()
batch_size = images.shape[0]
self.optimizer.zero_grad()
outputs = self.model(images)
 loss = self.criterion(outputs, targets)
 loss.backward()
final_scores.update(targets, outputs)
 summary_loss.update(loss.detach().item() , batch_size)
 self.optimizer.step()
if self.config.step_scheduler:
self.scheduler.step()
return summary_loss, final_scores
def save(self, path):
self.model.eval()
torch.save({
'model_state_dict': self.model.state_dict() ,
'optimizer_state_dict': self.optimizer.state_dict() ,
'scheduler_state_dict': self.scheduler.state_dict() ,
'best_summary_loss': self.best_summary_loss,
'epoch': self.epoch,
}, path)
 def load(self, path):
checkpoint = torch.load(path)
 self.model.load_state_dict(checkpoint['model_state_dict'])
 self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
 self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])
 self.best_summary_loss = checkpoint['best_summary_loss']
self.epoch = checkpoint['epoch'] + 1
def log(self, message):
if self.config.verbose:
print(message)
 with open(self.log_path, 'a+')as logger:
logger.write(f'{message}
' )
train=x
y_train=np.log1p(y)
 test=X_test
y=y_train<compute_train_metric>
def get_net() :
net = EfficientNet.from_pretrained('efficientnet-b2')
 net._fc = nn.Linear(in_features=1408, out_features=4, bias=True)
 return net
net = get_net().cuda()
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse = np.sqrt(-cross_val_score(model, train.values,y_train.values, scoring="neg_mean_squared_error", cv=kf))
return(rmse)
 def eval_model(model, name):
start_time = time.time()
score = rmsle_cv(model)
 print("{} score: {:.4f}({:.4f}), execution time: {:.1f}".format(name, score.mean() , score.std() , time.time() -start_time))<choose_model_class>
class TrainGlobalConfig:
num_workers = 4
batch_size = 16
n_epochs = 100
lr = 0.0001
verbose = True
verbose_step = 1
step_scheduler = False
validation_scheduler = True
SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau
scheduler_params = dict(
 mode='min',
factor=0.5,
patience=1,
verbose=False,
threshold=0.0001,
threshold_mode='abs',
cooldown=0,
min_lr=1e-8,
eps=1e-08 
)
mod_lasso = make_pipeline(RobustScaler() , Lasso(alpha=0.005, random_state=1))
eval_model(mod_lasso, "lasso")
 mod_enet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
eval_model(mod_enet, "enet")
 mod_cat = CatBoostRegressor(iterations=10000, learning_rate=0.01,
depth=5, eval_metric='RMSE',
colsample_bylevel=0.7, random_seed = 17, silent=True,
bagging_temperature = 0.2, early_stopping_rounds=200)
 eval_model(mod_cat, "cat")
 mod_gboost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=5, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state=5)
 eval_model(mod_gboost, "gboost")
 mod_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=10,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state=7, nthread=-1)
 eval_model(mod_xgb, "xgb")
 mod_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=1023,
learning_rate=0.05, n_estimators=650,
max_bin=58, bagging_fraction=0.80,max_depth=10,
bagging_freq=5, feature_fraction=0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf=7, min_sum_hessian_in_leaf=11)
 eval_model(mod_lgb, "lgb" )<train_model>
def run_training() :
device = torch.device('cuda:0')
 train_loader = torch.utils.data.DataLoader(
 train_dataset,
sampler=BalanceClassSampler(labels=train_dataset.get_labels() , mode="downsampling"),
batch_size=TrainGlobalConfig.batch_size,
pin_memory=False,
drop_last=True,
num_workers=TrainGlobalConfig.num_workers, 
)
 val_loader = torch.utils.data.DataLoader(
 validation_dataset,
batch_size=TrainGlobalConfig.batch_size,
num_workers=TrainGlobalConfig.num_workers,
shuffle=False,
sampler=SequentialSampler(validation_dataset),
pin_memory=False, 
)
 fitter = Fitter(model=net, device=device, config=TrainGlobalConfig)
 fitter.fit(train_loader, val_loader )
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features)
 mod_stacked = StackingAveragedModels(base_models =(mod_cat, mod_xgb, mod_gboost, mod_lgb), meta_model = mod_lasso)
 eval_model(mod_stacked, "stacked" )<predict_on_test>
file = open('.. /input/alaska2-public-baseline/log.txt', 'r')
 for line in file.readlines() :
print(line[:-1])
 file.close()
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def predict(model):
model.fit(train.values,y_train.values)
 train_pred = model.predict(train.values)
 pred = np.expm1(model.predict(test.values))
print(rmsle(y_train, train_pred))
return(pred )<predict_on_test>
checkpoint = torch.load('.. /input/alaska2-public-baseline/best-checkpoint-033epoch.bin')
 net.load_state_dict(checkpoint['model_state_dict']);
net.eval() ;
pre_lasso = predict(mod_lasso)
 pre_enet = predict(mod_enet)
 pre_xgb = predict(mod_xgb)
 pre_gboost = predict(mod_gboost)
 pre_lgb = predict(mod_lgb)
 pre_stack1 = predict(mod_stacked )<save_to_csv>
class DatasetSubmissionRetriever(Dataset):
def __init__(self, image_names, transforms=None):
super().__init__()
self.image_names = image_names
self.transforms = transforms
def __getitem__(self, index: int):
image_name = self.image_names[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/Test/{image_name}', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image_name, image
def __len__(self)-> int:
return self.image_names.shape[0]
new_test['id']=new_test.index
new_test['revenue']=pre_stack1
new_test[['id','revenue']].to_csv('submission_Dragon2.csv', index=False)
 new_test[['id','revenue']].head()<define_variables>
dataset = DatasetSubmissionRetriever(
 image_names=np.array([path.split('/')[-1] for path in glob('.. /input/alaska2-image-steganalysis/Test/*.jpg')]),
transforms=get_valid_transforms() , 
)
 data_loader = DataLoader(
 dataset,
batch_size=8,
shuffle=False,
num_workers=2,
drop_last=False,
)
OUTPUT_DIR = './'
MODEL_DIR = '.. /input/snap-mix/'
if not os.path.exists(OUTPUT_DIR):
os.makedirs(OUTPUT_DIR)
 TRAIN_PATH = '.. /input/cassava-leaf-disease-classification/train_images'
TEST_PATH = '.. /input/cassava-leaf-disease-classification/test_images'<init_hyperparams>
submission = pd.DataFrame(result)
 submission.to_csv('submission.csv', index=False)
 submission.head()