Kaggle competition: Cleaned vs Dirty
<SOS> metric: categorizationaccuracy Kaggle data source: cleaned-vs-dirty<drop_column>
!pip install -q albumentations
!pip install -q pretrainedmodels
!pip install -q kekas
!pip install -q adabound
train_raw['mvmtDistance'] = train_raw['walkDistance'] + train_raw['rideDistance']+train_raw['swimDistance']
features = list(train_raw.columns)
 features.remove("Id")
 features.remove("matchId")
 features.remove("groupId")
 features.remove("matchType")
 features.remove("winPlacePerc" )<merge>
%matplotlib inline
train_on_gpu = True
Blur, OpticalDistortion, GridDistortion, HueSaturationValue, Flip, VerticalFlip
print(os.listdir(".. /input/"))
sample_sub = pd.read_csv('.. /input/sample_submission.csv')
df_team = train_raw.copy()
df_max= train_raw.groupby(['matchId','groupId'])[features].agg('max')
 df_team = pd.merge(train_raw, df_max.reset_index() , suffixes=["", "_max"], how='left', on=['matchId', 'groupId'])
 df_team = df_team.drop(["assists_max","killPoints_max","headshotKills_max","numGroups_max","revives_max","teamKills_max","roadKills_max","vehicleDestroys_max"], axis=1)
 df_rank = df_max.groupby('matchId')[features].rank(pct=True ).reset_index()
df_team = pd.merge(df_team, df_rank, suffixes=["", "_maxRank"], how='left', on=['matchId', 'groupId'])
 df_team = df_team.drop(["roadKills_maxRank","matchDuration_maxRank","maxPlace_maxRank","numGroups_maxRank"], axis=1)
 del df_max
del df_rank
gc.collect()
df_sum = train_raw.groupby(['matchId','groupId'])[features].agg('sum')
 df_team = pd.merge(df_team, df_sum.reset_index() , suffixes=["", "_sum"], how='left', on=['matchId', 'groupId'])
 df_team = df_team.drop(["assists_sum","killPoints_sum","headshotKills_sum","numGroups_sum","revives_sum","teamKills_sum","roadKills_sum","vehicleDestroys_sum"], axis=1)
 del df_sum
gc.collect()<prepare_x_and_y>
train_folder = '.. /input/plates/plates/train/'
test_folder = '.. /input/plates/plates/test/'
cleaned = 'cleaned'
dirty = 'dirty'
train_clean = os.listdir(train_folder + cleaned)
 train_dirty = os.listdir(train_folder + dirty)
 test_files = os.listdir(test_folder)
 train_df = pd.concat([pd.DataFrame(train_clean, columns=['id']), pd.DataFrame(train_dirty, columns=['id'])])
 train_df.reset_index(drop=True, inplace=True)
 test_df = pd.DataFrame(test_files, columns=['id'])
 train_df['label'] = [1 if file_name.startswith('cleaned')else 0 for file_name in train_df['id']]
test_df['label'] = -1
print(train_df.head(3))
fig = plt.figure(figsize=(20,5))
for idx, img in enumerate(np.random.choice(train_df['id'], 10)) :
ax = fig.add_subplot(2, 10//2, idx+1, xticks=[], yticks=[])
 im = Image.open(train_folder + img.split('_')[0] + '/' + img)
 plt.imshow(im)
 lab = img
ax.set_title(f'{lab}' )
df_team = reduce_mem_usage(df_team)
 test_raw = reduce_mem_usage(test_raw)
 df_pred = df_team.drop(['Id','groupId','matchId'],axis=1)
 df_pred = reduce_mem_usage(df_pred)
 df_pred = pd.get_dummies(df_pred)
 df_pred_y = df_pred['winPlacePerc']
df_pred_x = df_pred.drop(['winPlacePerc'],axis=1 )<split>
def reader_train_fn(i, row):
if row['id'].split('_')[0] == cleaned:
image = cv2.imread(train_folder + cleaned + '/' + row['id'])[:,:,::-1]
else:
image = cv2.imread(train_folder + dirty + '/' + row['id'])[:,:,::-1]
label = torch.Tensor([row["label"]])
 return {"image": image, "label": label}
def reader_test_fn(i, row):
image = cv2.imread(test_folder + row['id'])[:,:,::-1]
label = torch.Tensor([row["label"]])
 return {"image": image, "label": label}
def step_fn(model: torch.nn.Module,
batch: torch.Tensor)-> torch.Tensor:
inp = batch["image"]
return model(inp)
 plt.imshow(reader_train_fn(32, train_df.loc[32])['image']);
X_train, X_test, y_train, y_test = train_test_split(df_pred_x, df_pred_y, test_size=0.33, random_state=42 )<init_hyperparams>
class Net(nn.Module):
def __init__(
 self,
num_classes: int,
p: float = 0.5,
pooling_size: int = 2,
last_conv_size: int = 2048,
arch: str = "squeezenet1_1",
pretrained: str = "imagenet")-> None:

 super().__init__()
net = pm.__dict__[arch](pretrained=pretrained)
 modules = list(net.children())[:-1]
modules += [nn.Sequential(
 AdaptiveConcatPool2d(size=pooling_size),
Flatten() ,
nn.BatchNorm1d(
 2 * pooling_size * pooling_size * last_conv_size 
),
nn.Dropout(p),
nn.Linear(
 2 * pooling_size * pooling_size * last_conv_size,
num_classes 
)
)]
self.net = nn.Sequential(*modules)
 def forward(self, x):
logits = self.net(x)
 return logits
params = {
'boosting_type':'gbdt',
'objective': 'regression',
'nthread': -1,
'verbose': 0,
'num_leaves': 31,
'learning_rate': 0.05,
'max_depth': -1,
'subsample': 0.8,
'subsample_freq': 1,
'colsample_bytree': 0.6,
'reg_aplha': 1,
'reg_lambda': 0.001,
'metric': 'rmse',
'min_split_gain': 0.5,
'min_child_weight': 1,
'min_child_samples': 10
}
train_set = lgb.Dataset(X_train, y_train, silent=True)
 model = lgb.train(params, train_set = train_set, num_boost_round=300)
 pred_test_y = model.predict(X_test, num_iteration = model.best_iteration)
 rms = sqrt(mean_squared_error(y_test, pred_test_y))
rms<compute_test_metric>
train, valid = train_test_split(train_df, stratify=train_df.label, test_size=0.15 )
rms = sqrt(mean_squared_error(y_test, prediction))
print(rms)
 df_result = pd.DataFrame(columns=['PRED','REAL'])
 df_result['PRED'] = prediction
df_result['REAL'] = y_test.reset_index(drop = True)
 df_result.head(100 )<merge>
train_tfms, val_tfms = get_transforms("image", 224, 0.5)
 train_dk = DataKek(df=train, reader_fn=reader_train_fn, transforms=train_tfms)
 val_dk = DataKek(df=valid, reader_fn=reader_train_fn, transforms=val_tfms)
 test_dk = DataKek(df=test_df, reader_fn=reader_test_fn, transforms=val_tfms)
 batch_size = 2
workers = 8
train_dl = DataLoader(train_dk, batch_size=batch_size, num_workers=workers, shuffle=True, drop_last=True)
 val_dl = DataLoader(val_dk, batch_size=batch_size, num_workers=workers, shuffle=False)
 test_dl = DataLoader(test_dk, batch_size=30, num_workers=workers, shuffle=False )
test_raw['mvmtDistance'] = test_raw['walkDistance'] + test_raw['rideDistance']+test_raw['swimDistance']
features = list(test_raw.columns)
 features.remove("Id")
 features.remove("matchId")
 features.remove("groupId")
 features.remove("matchType")
 df_max= test_raw.groupby(['matchId','groupId'])[features].agg('max')
 df_team = pd.merge(test_raw, df_max.reset_index() , suffixes=["", "_max"], how='left', on=['matchId', 'groupId'])
 df_team = df_team.drop(["assists_max","killPoints_max","headshotKills_max","numGroups_max","revives_max","teamKills_max","roadKills_max","vehicleDestroys_max"], axis=1)
 df_rank = df_max.groupby('matchId')[features].rank(pct=True ).reset_index()
df_team = pd.merge(df_team, df_rank, suffixes=["", "_maxRank"], how='left', on=['matchId', 'groupId'])
 df_team = df_team.drop(["roadKills_maxRank","matchDuration_maxRank","maxPlace_maxRank","numGroups_maxRank"], axis=1)
 del df_max
del df_rank
gc.collect()
df_sum = train_raw.groupby(['matchId','groupId'])[features].agg('sum')
 df_team = pd.merge(df_team, df_sum.reset_index() , suffixes=["", "_sum"], how='left', on=['matchId', 'groupId'])
 df_team = df_team.drop(["assists_sum","killPoints_sum","headshotKills_sum","numGroups_sum","revives_sum","teamKills_sum","roadKills_sum","vehicleDestroys_sum"], axis=1)
 del df_sum
gc.collect()
test = df_team.drop(['Id', 'groupId', 'matchId'],axis=1)
 test = pd.get_dummies(test)
 pred_test_y = model.predict(test, num_iteration = model.best_iteration)
 test = pd.DataFrame(columns=['Id', 'winPlacePerc'])
 test['Id'] = test_raw['Id']
test['winPlacePerc'] = pred_test_y
test.to_csv('submission.csv',index=False )<load_from_csv>
dataowner = DataOwner(train_dl, val_dl, None)
 BCEmodel = Net(num_classes=1)
 CEmodel = Net(num_classes=2)
 CEcriterion = nn.CrossEntropyLoss()
BCEcriterion = nn.BCEWithLogitsLoss()
df_sub = pd.read_csv(".. /input/dataset/score.csv" )<load_from_csv>
model = Net(num_classes=1, last_conv_size=1000)
 def bce_accuracy(target: torch.Tensor,
preds: torch.Tensor)-> float:
thresh = 0.5
target = target.cpu().detach().numpy()
preds =(torch.sigmoid(preds ).cpu().detach().numpy() > thresh ).astype(int)
 return accuracy_score(target, preds)
 BCE_keker = Keker(model=model,
dataowner=dataowner,
criterion=BCEcriterion,
step_fn=step_fn,
target_key="label",
metrics={"accuracy": bce_accuracy},
opt=adabound.AdaBound,
opt_params={'final_lr':0.001} )
df_test = pd.read_csv(".. /input/pubg-finish-placement-prediction/test_V2.csv" )<merge>
logdir = 'logs'
lrlogdir = 'lrlogs'
df_sub = df_sub.merge(df_test[["Id", "matchId", "groupId", "maxPlace", "numGroups"]], on="Id", how="left")
 df_sub_group = df_sub.groupby(["matchId", "groupId"] ).first().reset_index()
df_sub_group["rank"] = df_sub_group.groupby(["matchId"])["winPlacePerc"].rank()
df_sub_group = df_sub_group.merge(
 df_sub_group.groupby("matchId")["rank"].max().to_frame("max_rank" ).reset_index() ,
on="matchId", how="left")
 df_sub_group["adjusted_perc"] =(df_sub_group["rank"] - 1)/(df_sub_group["numGroups"] - 1)
 df_sub = df_sub.merge(df_sub_group[["adjusted_perc", "matchId", "groupId"]], on=["matchId", "groupId"], how="left")
 df_sub["winPlacePerc"] = df_sub["adjusted_perc"]<feature_engineering>
BCE_keker.freeze(model_attr='net')
 BCE_keker.kek(lr=0.0001,
epochs=50,
cp_saver_params={
"savedir": "bce_sueeze",
"n_best": 3,
"prefix": "bce_squeeze",
"mode": "min"},
logdir=logdir )
df_sub.loc[df_sub.maxPlace == 0, "winPlacePerc"] = 0
df_sub.loc[df_sub.maxPlace == 1, "winPlacePerc"] = 1<feature_engineering>
BCE_keker.load('bce_sueeze/bce_squeeze.best.h5')
 preds = BCE_keker.predict_loader(test_dl)
 test_df['label'] =(torch.sigmoid(torch.Tensor(preds)) > 0.5 ).numpy()
subset = df_sub.loc[df_sub.maxPlace > 1]
gap = 1.0 /(subset.maxPlace.values - 1)
 new_perc = np.around(subset.winPlacePerc.values / gap)* gap
df_sub.loc[df_sub.maxPlace > 1, "winPlacePerc"] = new_perc<feature_engineering>
test_df['id'] = test_df['id'].apply(lambda x: x.split('.')[0])
 test_df['label'] = test_df['label'].apply(lambda x: 'cleaned' if x==1 else 'dirty' )
<save_to_csv><EOS> 
test_df.to_csv('sub.csv', index=False )