Kaggle competition: ALASKA2 Image Steganalysis
<SOS> metric: WeightedAUC Kaggle data source: alaska2-image-steganalysis<create_dataframe>
!pip install -q efficientnet_pytorch
Compose, HorizontalFlip, CLAHE, HueSaturationValue,
RandomBrightness, RandomContrast, RandomGamma, OneOf, Resize,
ToFloat, ShiftScaleRotate, GridDistortion, RandomRotate90, Cutout,
RGBShift, RandomBrightness, RandomContrast, Blur, MotionBlur, MedianBlur, GaussNoise, CoarseDropout,
IAAAdditiveGaussianNoise, GaussNoise, OpticalDistortion, RandomSizedCrop, VerticalFlip 
)
train_stack = np.vstack([oof_lgb, oof_xgb, oof_cat, oof_lgb_1, oof_lgb_2] ).transpose()
train_stack = pd.DataFrame(train_stack, columns=['lgb', 'xgb', 'cat', 'lgb_1', 'lgb_2'])
 test_stack = np.vstack([prediction_lgb, prediction_xgb, prediction_cat, prediction_lgb_1, prediction_lgb_2] ).transpose()
test_stack = pd.DataFrame(test_stack, columns=['lgb', 'xgb', 'cat', 'lgb_1', 'lgb_2'] )<train_on_grid>
seed = 42
print(f'setting everything to seed {seed}')
 random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
model = linear_model.RidgeCV(alphas=(0.01, 0.1, 1.0, 10.0, 100.0), scoring='neg_mean_squared_error', cv=folds)
 oof_rcv_stack, prediction_rcv_stack = train_model(train_stack.values, test_stack.values, y, params=None, model_type='sklearn', model=model )<save_to_csv>
data_dir = '.. /input/alaska2-image-steganalysis'
folder_names = ['JMiPOD/', 'JUNIWARD/', 'UERD/']
class_names = ['Normal', 'JMiPOD_75', 'JMiPOD_90', 'JMiPOD_95',
'JUNIWARD_75', 'JUNIWARD_90', 'JUNIWARD_95',
'UERD_75', 'UERD_90', 'UERD_95']
class_labels = { name: i for i, name in enumerate(class_names)}
sub = pd.read_csv('.. /input/tmdb-box-office-prediction/sample_submission.csv')
 sub['revenue'] = np.expm1(prediction_lgb)
 sub.to_csv("lgb.csv", index=False)
 sub['revenue'] = np.expm1(( prediction_lgb + prediction_xgb)/ 2)
 sub.to_csv("blend.csv", index=False)
 sub['revenue'] = np.expm1(( prediction_lgb + prediction_xgb + prediction_cat)/ 3)
 sub.to_csv("blend1.csv", index=False)
 sub['revenue'] = np.expm1(( prediction_lgb + prediction_xgb + prediction_cat + prediction_lgb_1)/ 4)
 sub.to_csv("blend2.csv", index=False)
 sub['revenue'] = np.expm1(( prediction_lgb + prediction_xgb + prediction_cat + prediction_lgb_1 + prediction_lgb_2)/ 5)
 sub.to_csv("blend3.csv", index=False)
 sub['revenue'] = prediction_lgb_stack
sub.to_csv("stack_lgb.csv", index=False)
 sub['revenue'] = prediction_rcv_stack
sub.to_csv("stack_rcv.csv", index=False )<set_options>
train_df = pd.read_csv('.. /input/alaska2trainvalsplit/alaska2_train_df.csv')
 val_df = pd.read_csv('.. /input/alaska2trainvalsplit/alaska2_val_df.csv')
 print(train_df.sample(10))
train_df.Label.hist()
plt.title('Distribution of Classes' )
warnings.filterwarnings("ignore" )<feature_engineering>
class Alaska2Dataset(Dataset):
def __init__(self, df, augmentations=None):
self.data = df
self.augment = augmentations
def __len__(self):
return len(self.data)
 def __getitem__(self, idx):
fn, label = self.data.loc[idx]
im = cv2.imread(fn)[:, :, ::-1]
if self.augment:
im = self.augment(image=im)
 return im, label
img_size = 512
AUGMENTATIONS_TRAIN = Compose([
VerticalFlip(p=0.5),
HorizontalFlip(p=0.5),
RandomRotate90(p=0.5),
OneOf([
Cutout(max_h_size=50,max_w_size=50),
CoarseDropout(max_height=50, max_width=50)], p=0.5),
ToFloat(max_value=255),
ToTensor()
], p=1)
 AUGMENTATIONS_TEST = Compose([
ToFloat(max_value=255),
ToTensor()
], p=1)
def prepare(df):
global json_cols
global train_dict
df['rating'] = df['rating'].fillna(1.5)
 df['totalVotes'] = df['totalVotes'].fillna(6)
 df['weightedRating'] =(df['rating']*df['totalVotes'] + 6.367 * 1000)/(df['totalVotes'] + 1000)
 df[['release_month','release_day','release_year']]=df['release_date'].str.split('/',expand=True ).replace(np.nan, 0 ).astype(int)
 df['release_year'] = df['release_year']
df.loc[(df['release_year'] <= 18)&(df['release_year'] < 100), "release_year"] += 2000
df.loc[(df['release_year'] > 18)&(df['release_year'] < 100), "release_year"] += 1900
releaseDate = pd.to_datetime(df['release_date'])
 df['release_dayofweek'] = releaseDate.dt.dayofweek
df['release_quarter'] = releaseDate.dt.quarter
df['_budget_runtime_ratio'] = df['budget']/df['runtime']
df['_budget_popularity_ratio'] = df['budget']/df['popularity']
df['_budget_year_ratio'] = df['budget']/(df['release_year']*df['release_year'])
 df['_releaseYear_popularity_ratio'] = df['release_year']/df['popularity']
df['_releaseYear_popularity_ratio2'] = df['popularity']/df['release_year']
df['_popularity_totalVotes_ratio'] = df['totalVotes']/df['popularity']
df['_rating_popularity_ratio'] = df['rating']/df['popularity']
df['_rating_totalVotes_ratio'] = df['totalVotes']/df['rating']
df['_totalVotes_releaseYear_ratio'] = df['totalVotes']/df['release_year']
df['_budget_rating_ratio'] = df['budget']/df['rating']
df['_runtime_rating_ratio'] = df['runtime']/df['rating']
df['_budget_totalVotes_ratio'] = df['budget']/df['totalVotes']
df['has_homepage'] = 0
df.loc[pd.isnull(df['homepage']),"has_homepage"] = 1
df['isbelongs_to_collectionNA'] = 0
df.loc[pd.isnull(df['belongs_to_collection']),"isbelongs_to_collectionNA"] = 1
df['isTaglineNA'] = 0
df.loc[df['tagline'] == 0 ,"isTaglineNA"] = 1
df['isOriginalLanguageEng'] = 0
df.loc[ df['original_language'] == "en" ,"isOriginalLanguageEng"] = 1
df['isTitleDifferent'] = 1
df.loc[ df['original_title'] == df['title'] ,"isTitleDifferent"] = 0
df['isMovieReleased'] = 1
df.loc[ df['status'] != "Released" ,"isMovieReleased"] = 0
df['collection_id'] = df['belongs_to_collection'].apply(lambda x : np.nan if len(x)==0 else x[0]['id'])
 df['original_title_letter_count'] = df['original_title'].str.len()
df['original_title_word_count'] = df['original_title'].str.split().str.len()
df['title_word_count'] = df['title'].str.split().str.len()
df['overview_word_count'] = df['overview'].str.split().str.len()
df['tagline_word_count'] = df['tagline'].str.split().str.len()
df['production_countries_count'] = df['production_countries'].apply(lambda x : len(x))
df['production_companies_count'] = df['production_companies'].apply(lambda x : len(x))
df['cast_count'] = df['cast'].apply(lambda x : len(x))
df['crew_count'] = df['cast'].apply(lambda x : len(x))
df['meanruntimeByYear'] = df.groupby("release_year")["runtime"].aggregate('mean')
 df['meanPopularityByYear'] = df.groupby("release_year")["popularity"].aggregate('mean')
 df['meanBudgetByYear'] = df.groupby("release_year")["budget"].aggregate('mean')
 df['meantotalVotesByYear'] = df.groupby("release_year")["totalVotes"].aggregate('mean')
 df['meanTotalVotesByRating'] = df.groupby("rating")["totalVotes"].aggregate('mean')
 for col in ['genres', 'production_countries', 'spoken_languages', 'production_companies'] :
df[col] = df[col].map(lambda x: sorted(list(set([n if n in train_dict[col] else col+'_etc' for n in [d['name'] for d in x]])))).map(lambda x: ','.join(map(str, x)))
 temp = df[col].str.get_dummies(sep=',')
 df = pd.concat([df, temp], axis=1, sort=False)
 df.drop(['genres_etc'], axis = 1, inplace = True)
 df = df.drop(['id', 'revenue','belongs_to_collection','genres','homepage','imdb_id','overview','runtime'
,'poster_path','production_companies','production_countries','release_date','spoken_languages'
,'status','title','Keywords','cast','crew','original_language','original_title','tagline', 'collection_id'
],axis=1)
 df.fillna(value=0.0, inplace = True)
 return df<load_from_csv>
temp_df = train_df.sample(64 ).reset_index(drop=True)
 train_dataset = Alaska2Dataset(temp_df, augmentations=AUGMENTATIONS_TEST)
 batch_size = 64
num_workers = 0
temp_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=batch_size,
num_workers=num_workers, shuffle=False)
 images, labels = next(iter(temp_loader))
images = images['image'].permute(0, 2, 3, 1)
 max_images = 64
grid_width = 16
grid_height = int(max_images / grid_width)
 fig, axs = plt.subplots(grid_height, grid_width,
figsize=(grid_width+1, grid_height+2))
for i,(im, label)in enumerate(zip(images, labels)) :
ax = axs[int(i / grid_width), i % grid_width]
ax.imshow(im.squeeze())
 ax.set_title(str(label.item()))
ax.axis('off')
 plt.suptitle("0: Cover, 1: JMiPOD_75, 2: JMiPOD_90, 3: JMiPOD_95, 4: JUNIWARD_75, 5:JUNIWARD_90,
6: JUNIWARD_95, 7:UERD_75, 8:UERD_90, 9:UERD_95")
 plt.show()
del images
gc.collect()
train = pd.read_csv('.. /input/tmdb-box-office-prediction/train.csv')
 train['revenue'] = np.log1p(train['revenue'])
 y = train['revenue'].values
test = pd.read_csv('.. /input/tmdb-box-office-prediction/test.csv')
 test['revenue'] = np.nan
train = pd.merge(train, pd.read_csv('.. /input/tmdb-box-office-movies-rating-total-votes/trainRatingTotalVotes.csv'), how='left', on=['imdb_id'])
 test = pd.merge(test, pd.read_csv('.. /input/tmdb-box-office-movies-rating-total-votes/testRatingTotalVotes.csv'), how='left', on=['imdb_id'])
 json_cols = ['genres', 'production_companies', 'production_countries', 'spoken_languages', 'Keywords', 'cast', 'crew']
def get_dictionary(s):
try:
d = eval(s)
 except:
d = {}
return d
for col in tqdm(json_cols + ['belongs_to_collection']):
train[col] = train[col].apply(lambda x : get_dictionary(x))
test[col] = test[col].apply(lambda x : get_dictionary(x))
print(train.shape)
 train.head()<feature_engineering>
train_dataset = Alaska2Dataset(temp_df, augmentations=AUGMENTATIONS_TRAIN)
 batch_size = 64
num_workers = 0
temp_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=batch_size,
num_workers=num_workers, shuffle=False)
 images, labels = next(iter(temp_loader))
images = images['image'].permute(0, 2, 3, 1)
 max_images = 64
grid_width = 16
grid_height = int(max_images / grid_width)
 fig, axs = plt.subplots(grid_height, grid_width,
figsize=(grid_width+1, grid_height+2))
for i,(im, label)in enumerate(zip(images, labels)) :
ax = axs[int(i / grid_width), i % grid_width]
ax.imshow(im.squeeze())
 ax.set_title(str(label.item()))
ax.axis('off')
 plt.suptitle("0: Cover, 1: JMiPOD_75, 2: JMiPOD_90, 3: JMiPOD_95, 4: JUNIWARD_75, 5:JUNIWARD_90,
6: JUNIWARD_95, 7:UERD_75, 8:UERD_90, 9:UERD_95")
 plt.show()
del images, temp_df
gc.collect()
def get_json_dict(df):
global json_cols
result = dict()
for e_col in json_cols :
d = dict()
rows = df[e_col].values
for row in rows :
if row is None : continue
for i in row :
if i['name'] not in d :
d[i['name']] = 0
d[i['name']] += 1
result[e_col] = d
return result
train_dict = get_json_dict(train)
 test_dict = get_json_dict(test)
 for col in json_cols :
remove = []
train_id = set(list(train_dict[col].keys()))
test_id = set(list(test_dict[col].keys()))
remove += list(train_id - test_id)+ list(test_id - train_id)
 for i in train_id.union(test_id)- set(remove):
if train_dict[col][i] < 10 or i == '' :
remove += [i]
for i in remove :
if i in train_dict[col] :
del train_dict[col][i]
if i in test_dict[col] :
del test_dict[col][i]
print(col, 'size :', len(train_id.union(test_id)) , '->', len(train_dict[col]))<concatenate>
class Net(nn.Module):
def __init__(self, num_classes):
super().__init__()
self.model = EfficientNet.from_pretrained('efficientnet-b0')
 self.dense_output = nn.Linear(1280, num_classes)
 def forward(self, x):
feat = self.model.extract_features(x)
 feat = F.avg_pool2d(feat, feat.size() [2:] ).reshape(-1, 1280)
 return self.dense_output(feat )
all_data = prepare(pd.concat([train, test] ).reset_index(drop = True))
train = all_data.loc[:train.shape[0] - 1,:]
test = all_data.loc[train.shape[0]:,:]
print(train.shape)
 train.head()<choose_model_class>
batch_size = 8
num_workers = 8
train_dataset = Alaska2Dataset(train_df, augmentations=AUGMENTATIONS_TRAIN)
 valid_dataset = Alaska2Dataset(val_df.sample(1000 ).reset_index(drop=True), augmentations=AUGMENTATIONS_TEST)
 train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=batch_size,
num_workers=num_workers,
shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid_dataset,
batch_size=batch_size*2,
num_workers=num_workers,
shuffle=False)
 device = 'cuda'
model = Net(num_classes=len(class_labels)).to(device)
 model.load_state_dict(torch.load('.. /input/alaska2trainvalsplit/val_loss_6.08_auc_0.875.pth'))
optimizer = torch.optim.AdamW(model.parameters() , lr=1e-6 )
random_seed = 2019
k = 5
fold = list(KFold(k, shuffle = True, random_state = random_seed ).split(train))
np.random.seed(random_seed )<train_model>
def alaska_weighted_auc(y_true, y_valid):
tpr_thresholds = [0.0, 0.4, 1.0]
weights = [2, 1]
fpr, tpr, thresholds = metrics.roc_curve(y_true, y_valid, pos_label=1)
 areas = np.array(tpr_thresholds[1:])- np.array(tpr_thresholds[:-1])
 normalization = np.dot(areas, weights)
 competition_metric = 0
for idx, weight in enumerate(weights):
y_min = tpr_thresholds[idx]
y_max = tpr_thresholds[idx + 1]
mask =(y_min < tpr)&(tpr < y_max)
 x_padding = np.linspace(fpr[mask][-1], 1, 100)
 x = np.concatenate([fpr[mask], x_padding])
 y = np.concatenate([tpr[mask], [y_max] * len(x_padding)])
 y = y - y_min
score = metrics.auc(x, y)
 submetric = score * weight
best_subscore =(y_max - y_min)* weight
competition_metric += submetric
return competition_metric / normalization
def xgb_model(trn_x, trn_y, val_x, val_y, test, verbose):
params = {'objective': 'reg:linear',
'eta': 0.01,
'max_depth': 6,
'subsample': 0.7,
'colsample_bytree': 0.8,
'eval_metric': 'rmse',
'seed': random_seed,
'silent': True,
}
record = dict()
model = xgb.train(params
, xgb.DMatrix(trn_x, trn_y)
 , 10000
, [(xgb.DMatrix(trn_x, trn_y), 'train'),(xgb.DMatrix(val_x, val_y), 'valid')]
, verbose_eval=verbose
, early_stopping_rounds=200
, callbacks = [xgb.callback.record_evaluation(record)])
 best_idx = np.argmin(np.array(record['valid']['rmse']))
val_pred = model.predict(xgb.DMatrix(val_x), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(test), ntree_limit=model.best_ntree_limit)
 return {'val':val_pred, 'test':test_pred, 'error':record['valid']['rmse'][best_idx], 'importance':[i for k, i in model.get_score().items() ]}<train_model>
criterion = torch.nn.CrossEntropyLoss()
num_epochs = 2
train_loss, val_loss = [], []
for epoch in range(num_epochs):
print('Epoch {}/{}'.format(epoch, num_epochs - 1))
print('-' * 10)
 model.train()
running_loss = 0
tk0 = tqdm(train_loader, total=int(len(train_loader)))
 for im, labels in tk0:
inputs = im["image"].to(device, dtype=torch.float)
 labels = labels.to(device, dtype=torch.long)
 optimizer.zero_grad()
outputs = model(inputs)
 loss = criterion(outputs, labels)
 loss.backward()
optimizer.step()
running_loss += loss.item()
tk0.set_postfix(loss=(loss.item()))
epoch_loss = running_loss /(len(train_loader)/batch_size)
 train_loss.append(epoch_loss)
 print('Training Loss: {:.8f}'.format(epoch_loss))
tk1 = tqdm(valid_loader, total=int(len(valid_loader)))
 model.eval()
running_loss = 0
y, preds = [], []
with torch.no_grad() :
for(im, labels)in tk1:
inputs = im["image"].to(device, dtype=torch.float)
 labels = labels.to(device, dtype=torch.long)
 outputs = model(inputs)
 loss = criterion(outputs, labels)
 y.extend(labels.cpu().numpy().astype(int))
preds.extend(F.softmax(outputs, 1 ).cpu().numpy())
 running_loss += loss.item()
tk1.set_postfix(loss=(loss.item()))
epoch_loss = running_loss /(len(valid_loader)/batch_size)
 val_loss.append(epoch_loss)
 preds = np.array(preds)
 y = np.array(y)
 labels = preds.argmax(1)
 for class_label in np.unique(y):
idx = y == class_label
acc =(labels[idx] == y[idx] ).astype(np.float ).mean() *100
print('accuracy for class', class_names[class_label], 'is', acc)
 acc =(labels == y ).mean() *100
new_preds = np.zeros(( len(preds),))
temp = preds[labels != 0, 1:]
new_preds[labels != 0] = temp.sum(1)
 new_preds[labels == 0] = 1 - preds[labels == 0, 0]
y = np.array(y)
 y[y != 0] = 1
auc_score = alaska_weighted_auc(y, new_preds)
 print(
 f'Val Loss: {epoch_loss:.3}, Weighted AUC:{auc_score:.3}, Acc: {acc:.3}')
 torch.save(model.state_dict() ,
f"epoch_{epoch}_val_loss_{epoch_loss:.3}_auc_{auc_score:.3}.pth")
def lgb_model(trn_x, trn_y, val_x, val_y, test, verbose):
params = {'objective':'regression',
'num_leaves' : 40,
'min_data_in_leaf' : 20,
'max_depth' : 4,
'learning_rate': 0.01,
"feature_fraction": 0.8,
"bagging_freq": 1,
"bagging_fraction": 0.8,
"bagging_seed": random_seed,
"metric": 'rmse',
"random_state" : random_seed,
"verbosity": -1}
record = dict()
model = lgb.train(params
, lgb.Dataset(trn_x, trn_y)
 , num_boost_round = 10000
, valid_sets = [lgb.Dataset(val_x, val_y)]
, verbose_eval = verbose
, early_stopping_rounds = 200
, callbacks = [lgb.record_evaluation(record)] 
)
 best_idx = np.argmin(np.array(record['valid_0']['rmse']))
val_pred = model.predict(val_x, num_iteration = model.best_iteration)
 test_pred = model.predict(test, num_iteration = model.best_iteration)
 return {'val':val_pred, 'test':test_pred, 'error':record['valid_0']['rmse'][best_idx], 'importance':model.feature_importance('gain')}<train_on_grid>
class Alaska2TestDataset(Dataset):
def __init__(self, df, augmentations=None):
self.data = df
self.augment = augmentations
def __len__(self):
return len(self.data)
 def __getitem__(self, idx):
fn = self.data.loc[idx][0]
im = cv2.imread(fn)[:, :, ::-1]
if self.augment:
im = self.augment(image=im)
 return im
test_filenames = sorted(glob(f"{data_dir}/Test/*.jpg"))
test_df = pd.DataFrame({'ImageFileName': list(
 test_filenames)}, columns=['ImageFileName'])
 batch_size = 16
num_workers = 4
test_dataset = Alaska2TestDataset(test_df, augmentations=AUGMENTATIONS_TEST)
 test_loader = torch.utils.data.DataLoader(test_dataset,
batch_size=batch_size,
num_workers=num_workers,
shuffle=False,
drop_last=False )
<train_model><EOS> 
model.eval()
preds = []
tk0 = tqdm(test_loader)
 with torch.no_grad() :
for i, im in enumerate(tk0):
inputs = im["image"].to(device)
 im = inputs.flip(2)
 outputs = model(im)
 im = inputs.flip(3)
 outputs =(0.25*outputs + 0.25*model(im))
outputs =(outputs + 0.5*model(inputs))
preds.extend(F.softmax(outputs, 1 ).cpu().numpy())
 preds = np.array(preds)
 labels = preds.argmax(1)
 new_preds = np.zeros(( len(preds),))
new_preds[labels != 0] = preds[labels != 0, 1:].sum(1)
 new_preds[labels == 0] = 1 - preds[labels == 0, 0]
test_df['Id'] = test_df['ImageFileName'].apply(lambda x: x.split(os.sep)[-1])
 test_df['Label'] = new_preds
test_df = test_df.drop('ImageFileName', axis=1)
 test_df.to_csv('submission_eb0.csv', index=False)
 print(test_df.head() )