Kaggle competition: ALASKA2 Image Steganalysis
<SOS> metric: WeightedAUC Kaggle data source: alaska2-image-steganalysis<define_variables>
!pip install -q efficientnet_pytorch > /dev/null
features = features+olang+train_tagline_keyword_w_cols+train_genres_w_cols+train_belongs_to_collection_w_cols \
+train_production_companies_w_cols + train_crew_w_cols+ train_production_countries_w_cols<prepare_x_and_y>
SEED = 42
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(SEED )
target_column = 'revenue'
columns_for_prediction=features
X = train[columns_for_prediction].copy()
y_scale = preprocessing.MinMaxScaler()
y = y_scale.fit_transform(train[target_column].values.reshape(-1, 1))
X_unseen = test[columns_for_prediction].copy()
scale = preprocessing.StandardScaler()
X = pd.DataFrame(scale.fit_transform(X),columns=columns_for_prediction)
 X_unseen = pd.DataFrame(scale.transform(test[columns_for_prediction]),columns=columns_for_prediction)
 budget_min = X['budget'].quantile(0.28)
 X['budget'] = X['budget'].replace(0,budget_min)
 X_unseen['budget'] = X_unseen['budget'].replace(0,budget_min)
<train_model>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.1, random_state=2019)
 params = {'n_estimators': 700, 'max_depth': 4, 'min_samples_split': 2,
'learning_rate': 0.01, 'loss': 'ls'}
reg = GradientBoostingRegressor(**params ).fit(X_train, y_train)
 score = reg.score(X_test, y_test)
 print('Test score %d'%score)
 preds = reg.predict(X_test)
 err = mean_squared_error(y_test, preds)
 print('Test mse %d'%err)
 reg = GradientBoostingRegressor(n_estimators=700 ).fit(X, y)
 score = reg.score(X, y)
 print('Train score %d'%score)
 preds_first = reg.predict(X_unseen )<choose_model_class>
def get_train_transforms() :
return A.Compose([
A.HorizontalFlip(p=0.5),
A.VerticalFlip(p=0.5),
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
], p=1.0)
 def get_valid_transforms() :
return A.Compose([
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )
<split>
DATA_ROOT_PATH = '.. /input/alaska2-image-steganalysis'
def onehot(size, target):
vec = torch.zeros(size, dtype=torch.float32)
 vec[target] = 1.
 return vec
class DatasetRetriever(Dataset):
def __init__(self, kinds, image_names, labels, transforms=None):
super().__init__()
self.kinds = kinds
self.image_names = image_names
self.labels = labels
self.transforms = transforms
def __getitem__(self, index: int):
kind, image_name, label = self.kinds[index], self.image_names[index], self.labels[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{kind}/{image_name}', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
target = onehot(4, label)
 return image, target
def __len__(self)-> int:
return self.image_names.shape[0]
def get_labels(self):
return list(self.labels )
<save_to_csv>
fold_number = 0
train_dataset = DatasetRetriever(
 kinds=dataset[dataset['fold'] != fold_number].kind.values,
image_names=dataset[dataset['fold'] != fold_number].image_name.values,
labels=dataset[dataset['fold'] != fold_number].label.values,
transforms=get_train_transforms() , 
)
 validation_dataset = DatasetRetriever(
 kinds=dataset[dataset['fold'] == fold_number].kind.values,
image_names=dataset[dataset['fold'] == fold_number].image_name.values,
labels=dataset[dataset['fold'] == fold_number].label.values,
transforms=get_valid_transforms() ,
)
median_revenue = train[target_column].median()
preds = preds_estop
preds = y_scale.inverse_transform(preds.reshape(-1, 1))
preds[preds < 0] = median_revenue
subm['revenue'] = preds
subm.to_csv('submission.csv', index=False)
 print(subm.head() )<feature_engineering>
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
def alaska_weighted_auc(y_true, y_valid):

 tpr_thresholds = [0.0, 0.4, 1.0]
weights = [2, 1]
fpr, tpr, thresholds = metrics.roc_curve(y_true, y_valid, pos_label=1)
 areas = np.array(tpr_thresholds[1:])- np.array(tpr_thresholds[:-1])
 normalization = np.dot(areas, weights)
 competition_metric = 0
for idx, weight in enumerate(weights):
y_min = tpr_thresholds[idx]
y_max = tpr_thresholds[idx + 1]
mask =(y_min < tpr)&(tpr < y_max)
 x_padding = np.linspace(fpr[mask][-1], 1, 100)
 x = np.concatenate([fpr[mask], x_padding])
 y = np.concatenate([tpr[mask], [y_max] * len(x_padding)])
 y = y - y_min
score = metrics.auc(x, y)
 submetric = score * weight
best_subscore =(y_max - y_min)* weight
competition_metric += submetric
return competition_metric / normalization
class RocAucMeter(object):
def __init__(self):
self.reset()
def reset(self):
self.y_true = np.array([0,1])
 self.y_pred = np.array([0.5,0.5])
 self.score = 0
def update(self, y_true, y_pred):
y_true = y_true.cpu().numpy().argmax(axis=1 ).clip(min=0, max=1 ).astype(int)
 y_pred = 1 - nn.functional.softmax(y_pred, dim=1 ).data.cpu().numpy() [:,0]
self.y_true = np.hstack(( self.y_true, y_true))
self.y_pred = np.hstack(( self.y_pred, y_pred))
self.score = alaska_weighted_auc(self.y_true, self.y_pred)
 @property
def avg(self):
return self.score
test['revenue'] = subm['revenue']<set_options>
class LabelSmoothing(nn.Module):
def __init__(self, smoothing = 0.05):
super(LabelSmoothing, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
def forward(self, x, target):
if self.training:
x = x.float()
target = target.float()
logprobs = torch.nn.functional.log_softmax(x, dim = -1)
 nll_loss = -logprobs * target
nll_loss = nll_loss.sum(-1)
 smooth_loss = -logprobs.mean(dim=-1)
 loss = self.confidence * nll_loss + self.smoothing * smooth_loss
return loss.mean()
else:
return torch.nn.functional.cross_entropy(x, target )
pd.set_option('max_columns', None)
 %matplotlib inline
plt.style.use('ggplot')
 stop = set(stopwords.words('english'))
warnings.filterwarnings("ignore" )<load_from_csv>
warnings.filterwarnings("ignore")
 class Fitter:
def __init__(self, model, device, config):
self.config = config
self.epoch = 0
self.base_dir = './'
self.log_path = f'{self.base_dir}/log.txt'
self.best_summary_loss = 10**5
self.model = model
self.device = device
param_optimizer = list(self.model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
self.optimizer = torch.optim.AdamW(self.model.parameters() , lr=config.lr)
 self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)
 self.criterion = LabelSmoothing().to(self.device)
 self.log(f'Fitter prepared.Device is {self.device}')
 def fit(self, train_loader, validation_loader):
for e in range(self.config.n_epochs):
if self.config.verbose:
lr = self.optimizer.param_groups[0]['lr']
timestamp = datetime.utcnow().isoformat()
self.log(f'
{timestamp}
LR: {lr}')
 t = time.time()
summary_loss, final_scores = self.train_one_epoch(train_loader)
 self.log(f'[RESULT]: Train.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, final_score: {final_scores.avg:.5f}, time: {(time.time() - t):.5f}')
 self.save(f'{self.base_dir}/last-checkpoint.bin')
 t = time.time()
summary_loss, final_scores = self.validation(validation_loader)
 self.log(f'[RESULT]: Val.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, final_score: {final_scores.avg:.5f}, time: {(time.time() - t):.5f}')
 if summary_loss.avg < self.best_summary_loss:
self.best_summary_loss = summary_loss.avg
self.model.eval()
self.save(f'{self.base_dir}/best-checkpoint-{str(self.epoch ).zfill(3)}epoch.bin')
 for path in sorted(glob(f'{self.base_dir}/best-checkpoint-*epoch.bin')) [:-3]:
os.remove(path)
 if self.config.validation_scheduler:
self.scheduler.step(metrics=summary_loss.avg)
 self.epoch += 1
def validation(self, val_loader):
self.model.eval()
summary_loss = AverageMeter()
final_scores = RocAucMeter()
t = time.time()
for step,(images, targets)in enumerate(val_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Val Step {step}/{len(val_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, final_score: {final_scores.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 with torch.no_grad() :
targets = targets.to(self.device ).float()
batch_size = images.shape[0]
images = images.to(self.device ).float()
outputs = self.model(images)
 loss = self.criterion(outputs, targets)
 final_scores.update(targets, outputs)
 summary_loss.update(loss.detach().item() , batch_size)
 return summary_loss, final_scores
def train_one_epoch(self, train_loader):
self.model.train()
summary_loss = AverageMeter()
final_scores = RocAucMeter()
t = time.time()
for step,(images, targets)in enumerate(train_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Train Step {step}/{len(train_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, final_score: {final_scores.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 targets = targets.to(self.device ).float()
images = images.to(self.device ).float()
batch_size = images.shape[0]
self.optimizer.zero_grad()
outputs = self.model(images)
 loss = self.criterion(outputs, targets)
 loss.backward()
final_scores.update(targets, outputs)
 summary_loss.update(loss.detach().item() , batch_size)
 self.optimizer.step()
if self.config.step_scheduler:
self.scheduler.step()
return summary_loss, final_scores
def save(self, path):
self.model.eval()
torch.save({
'model_state_dict': self.model.state_dict() ,
'optimizer_state_dict': self.optimizer.state_dict() ,
'scheduler_state_dict': self.scheduler.state_dict() ,
'best_summary_loss': self.best_summary_loss,
'epoch': self.epoch,
}, path)
 def load(self, path):
checkpoint = torch.load(path)
 self.model.load_state_dict(checkpoint['model_state_dict'])
 self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
 self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])
 self.best_summary_loss = checkpoint['best_summary_loss']
self.epoch = checkpoint['epoch'] + 1
def log(self, message):
if self.config.verbose:
print(message)
 with open(self.log_path, 'a+')as logger:
logger.write(f'{message}
' )
train=pd.read_csv('.. /input/train.csv',index_col=0)
 test=pd.read_csv('.. /input/test.csv',index_col=0 )<categorify>
def get_net() :
net = EfficientNet.from_pretrained('efficientnet-b2')
 net._fc = nn.Linear(in_features=1408, out_features=4, bias=True)
 return net
net = get_net().cuda()
!pip install --upgrade pip
!pip install imdbpy
ia=IMDb()
def fill_col(df,col,key,fill_missing_type,nested_arr):
indexes=df.index[df[col]!=df[col]].tolist()
for index in indexes:
imdb_id=df.loc[index]['imdb_id']
movie=ia.get_movie(imdb_id[2:])
 if key in movie:
if(nested_arr):
arr=[]
for el in movie[key]:
arr.append(str(el))
df.at[index, col] = ','.join(arr)
 else:
df.at[index, col] = ','.join(movie[key])
 else:
if(fill_missing_type=='mean'):
df.at[index, col] = df[col].mean()
elif(fill_missing_type=='others'):
df.at[index, col] = 'others'
elif(fill_missing_type=='None'):
df.at[index, col] = None
fill_col(train,'runtime','runtimes','None',False)
 fill_col(test,'runtime','runtimes','None',False )<feature_engineering>
class TrainGlobalConfig:
num_workers = 4
batch_size = 16
n_epochs = 25
lr = 0.001
verbose = True
verbose_step = 1
step_scheduler = False
validation_scheduler = True
SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau
scheduler_params = dict(
 mode='min',
factor=0.5,
patience=1,
verbose=False,
threshold=0.0001,
threshold_mode='abs',
cooldown=0,
min_lr=1e-8,
eps=1e-08 
)
test['revenue']=-99<feature_engineering>
def run_training() :
device = torch.device('cuda:0')
 train_loader = torch.utils.data.DataLoader(
 train_dataset,
sampler=BalanceClassSampler(labels=train_dataset.get_labels() , mode="downsampling"),
batch_size=TrainGlobalConfig.batch_size,
pin_memory=False,
drop_last=True,
num_workers=TrainGlobalConfig.num_workers, 
)
 val_loader = torch.utils.data.DataLoader(
 validation_dataset,
batch_size=TrainGlobalConfig.batch_size,
num_workers=TrainGlobalConfig.num_workers,
shuffle=False,
sampler=SequentialSampler(validation_dataset),
pin_memory=False, 
)
 fitter = Fitter(model=net, device=device, config=TrainGlobalConfig)
 fitter.fit(train_loader, val_loader )
train['be_is_na']=train['belongs_to_collection'].apply(lambda x :1 if x is np.nan else 0)
 test['be_is_na']=test['belongs_to_collection'].apply(lambda x :1 if x is np.nan else 0)
 train=train.drop('belongs_to_collection',axis=1)
 test=test.drop('belongs_to_collection',axis=1 )<feature_engineering>
file = open('.. /input/alaska2-public-baseline/log.txt', 'r')
 for line in file.readlines() :
print(line[:-1])
 file.close()
train.loc[train.index == 16,'revenue'] = 192864
train.loc[train.index == 90,'budget'] = 30000000
train.loc[train.index== 118,'budget'] = 60000000
train.loc[train.index== 149,'budget'] = 18000000
train.loc[train.index== 313,'revenue'] = 12000000
train.loc[train.index == 451,'revenue'] = 12000000
train.loc[train.index == 464,'budget'] = 20000000
train.loc[train.index == 470,'budget'] = 13000000
train.loc[train.index== 513,'budget'] = 930000
train.loc[train.index == 797,'budget'] = 8000000
train.loc[train.index == 819,'budget'] = 90000000
train.loc[train.index == 850,'budget'] = 90000000
train.loc[train.index == 1007,'budget'] = 2
train.loc[train.index== 1112,'budget'] = 7500000
train.loc[train.index == 1131,'budget'] = 4300000
train.loc[train.index == 1359,'budget'] = 10000000
train.loc[train.index == 1542,'budget'] = 1
train.loc[train.index == 1570,'budget'] = 15800000
train.loc[train.index== 1571,'budget'] = 4000000
train.loc[train.index == 1714,'budget'] = 46000000
train.loc[train.index == 1721,'budget'] = 17500000
train.loc[train.index== 1865,'revenue'] = 25000000
train.loc[train.index == 1885,'budget'] = 12
train.loc[train.index == 2091,'budget'] = 10
train.loc[train.index == 2268,'budget'] = 17500000
train.loc[train.index == 2491,'budget'] = 6
train.loc[train.index == 2602,'budget'] = 31000000
train.loc[train.index == 2612,'budget'] = 15000000
train.loc[train.index == 2696,'budget'] = 10000000
train.loc[train.index == 2801,'budget'] = 10000000
train.loc[train.index == 335,'budget'] = 2
train.loc[train.index == 348,'budget'] = 12
train.loc[train.index == 470,'budget'] = 13000000
train.loc[train.index == 513,'budget'] = 1100000
train.loc[train.index == 640,'budget'] = 6
train.loc[train.index == 696,'budget'] = 1
train.loc[train.index == 797,'budget'] = 8000000
train.loc[train.index == 850,'budget'] = 1500000
train.loc[train.index == 1199,'budget'] = 5
train.loc[train.index == 1282,'budget'] = 9
train.loc[train.index== 1347,'budget'] = 1
train.loc[train.index== 1755,'budget'] = 2
train.loc[train.index == 1801,'budget'] = 5
train.loc[train.index == 1918,'budget'] = 592
train.loc[train.index == 2033,'budget'] = 4
train.loc[train.index == 2118,'budget'] = 344
train.loc[train.index == 2252,'budget'] = 130
train.loc[train.index == 2256,'budget'] = 1
train.loc[train.index == 2696,'budget'] = 10000000<feature_engineering>
checkpoint = torch.load('.. /input/alaska2-public-baseline/best-checkpoint-033epoch.bin')
 net.load_state_dict(checkpoint['model_state_dict']);
net.eval() ;
test.loc[test.index== 3033,'budget'] = 250
test.loc[test.index== 3051,'budget'] = 50
test.loc[test.index == 3084,'budget'] = 337
test.loc[test.index == 3224,'budget'] = 4
test.loc[test.index == 3594,'budget'] = 25
test.loc[test.index == 3619,'budget'] = 500
test.loc[test.index == 3831,'budget'] = 3
test.loc[test.index== 3935,'budget'] = 500
test.loc[test.index == 4049,'budget'] = 995946
test.loc[test.index== 4424,'budget'] = 3
test.loc[test.index == 4460,'budget'] = 8
test.loc[test.index == 4555,'budget'] = 1200000
test.loc[test.index== 4624,'budget'] = 30
test.loc[test.index== 4645,'budget'] = 500
test.loc[test.index == 4709,'budget'] = 450
test.loc[test.index == 4839,'budget'] = 7
test.loc[test.index== 3125,'budget'] = 25
test.loc[test.index== 3142,'budget'] = 1
test.loc[test.index == 3201,'budget'] = 450
test.loc[test.index == 3222,'budget'] = 6
test.loc[test.index== 3545,'budget'] = 38
test.loc[test.index == 3670,'budget'] = 18
test.loc[test.index == 3792,'budget'] = 19
test.loc[test.index == 3881,'budget'] = 7
test.loc[test.index == 3969,'budget'] = 400
test.loc[test.index == 4196,'budget'] = 6
test.loc[test.index == 4221,'budget'] = 11
test.loc[test.index == 4222,'budget'] = 500
test.loc[test.index== 4285,'budget'] = 11
test.loc[test.index == 4319,'budget'] = 1
test.loc[test.index == 4639,'budget'] = 10
test.loc[test.index == 4719,'budget'] = 45
test.loc[test.index == 4822,'budget'] = 22
test.loc[test.index == 4829,'budget'] = 20
test.loc[test.index== 4969,'budget'] = 20
test.loc[test.index== 5021,'budget'] = 40
test.loc[test.index== 5035,'budget'] = 1
test.loc[test.index== 5063,'budget'] = 14
test.loc[test.index == 5119,'budget'] = 2
test.loc[test.index== 5214,'budget'] = 30
test.loc[test.index== 5221,'budget'] = 50
test.loc[test.index== 4903,'budget'] = 15
test.loc[test.index == 4983,'budget'] = 3
test.loc[test.index == 5102,'budget'] = 28
test.loc[test.index== 5217,'budget'] = 75
test.loc[test.index == 5224,'budget'] = 3
test.loc[test.index== 5469,'budget'] = 20
test.loc[test.index == 5840,'budget'] = 1
test.loc[test.index == 5960,'budget'] = 30
test.loc[test.index == 6506,'budget'] = 11
test.loc[test.index== 6553,'budget'] = 280
test.loc[test.index == 6561,'budget'] = 7
test.loc[test.index== 6582,'budget'] = 218
test.loc[test.index == 6638,'budget'] = 5
test.loc[test.index== 6749,'budget'] = 8
test.loc[test.index==6759,'budget'] = 50
test.loc[test.index == 6856,'budget'] = 10
test.loc[test.index== 6858,'budget'] = 100
test.loc[test.index == 6876,'budget'] = 250
test.loc[test.index == 6972,'budget'] = 1
test.loc[test.index== 7079,'budget'] = 8000000
test.loc[test.index == 7150,'budget'] = 118
test.loc[test.index == 6506,'budget'] = 118
test.loc[test.index == 7225,'budget'] = 6
test.loc[test.index == 7231,'budget'] = 85
test.loc[test.index == 5222,'budget'] = 5
test.loc[test.index == 5322,'budget'] = 90
test.loc[test.index == 5350,'budget'] = 70
test.loc[test.index == 5378,'budget'] = 10
test.loc[test.index== 5545,'budget'] = 80
test.loc[test.index == 5810,'budget'] = 8
test.loc[test.index== 5926,'budget'] = 300
test.loc[test.index== 5927,'budget'] = 4
test.loc[test.index== 5986,'budget'] = 1
test.loc[test.index == 6053,'budget'] = 20
test.loc[test.index== 6104,'budget'] = 1
test.loc[test.index == 6130,'budget'] = 30
test.loc[test.index == 6301,'budget'] = 150
test.loc[test.index == 6276,'budget'] = 100
test.loc[test.index == 6473,'budget'] = 100
test.loc[test.index== 6842,'budget'] = 30<concatenate>
class DatasetSubmissionRetriever(Dataset):
def __init__(self, image_names, transforms=None):
super().__init__()
self.image_names = image_names
self.transforms = transforms
def __getitem__(self, index: int):
image_name = self.image_names[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/Test/{image_name}', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image_name, image
def __len__(self)-> int:
return self.image_names.shape[0]
new_data=pd.concat([train,test],axis=0)
 new_data.head()<count_missing_values>
dataset = DatasetSubmissionRetriever(
 image_names=np.array([path.split('/')[-1] for path in glob('.. /input/alaska2-image-steganalysis/Test/*.jpg')]),
transforms=get_valid_transforms() , 
)
 data_loader = DataLoader(
 dataset,
batch_size=8,
shuffle=False,
num_workers=2,
drop_last=False,
)
<filter><EOS> 
submission = pd.DataFrame(result)
 submission.to_csv('submission.csv', index=False)
 submission.head()