Kaggle competition: Categorical Feature Encoding Challenge
ROOT = Path.cwd().parent
INPUT = ROOT / "input"
OUTPUT = ROOT / "output"
DATA = INPUT / "ranzcr-clip-catheter-line-classification"
TRAIN = DATA / "train"
TEST = DATA / "test"
TRAINED_MODEL = INPUT / "ranzcr-clip-weights-for-multi-head-model-v2"
TMP = ROOT / "tmp"
TMP.mkdir(exist_ok=True)
 RANDAM_SEED = 1086
N_CLASSES = 11
FOLDS = [0, 1, 2, 3, 4]
N_FOLD = len(FOLDS)
 IMAGE_SIZE =(640, 640)
 CONVERT_TO_RANK = True
FAST_COMMIT = False
CLASSES = [
'ETT - Abnormal',
'ETT - Borderline',
'ETT - Normal',
'NGT - Abnormal',
'NGT - Borderline',
'NGT - Incompletely Imaged',
'NGT - Normal',
'CVC - Abnormal',
'CVC - Borderline',
'CVC - Normal',
'Swan Ganz Catheter Present'
]<load_from_csv>
train = pd.read_csv('.. /input/cat-in-the-dat/train.csv', index_col='id')
 test = pd.read_csv('.. /input/cat-in-the-dat/test.csv', index_col='id' )
for p in DATA.iterdir() :
print(p.name)
 train = pd.read_csv(DATA / "train.csv")
 smpl_sub = pd.read_csv(DATA / "sample_submission.csv" )<split>
train['ord_5_1'] = train['ord_5'].str[0]
train['ord_5_2'] = train['ord_5'].str[1]
train = train.drop('ord_5', axis=1)
 test['ord_5_1'] = test['ord_5'].str[0]
test['ord_5_2'] = test['ord_5'].str[1]
test = test.drop('ord_5', axis=1 )
if FAST_COMMIT and len(smpl_sub)== 3582:
smpl_sub = smpl_sub.iloc[:64 * 3].reset_index(drop=True )<categorify>
columns_to_test = ['nom_7', 'nom_8', 'nom_9']
replace_xor = lambda x: 'xor' if x in xor_values else x
for column in columns_to_test:
xor_values = set(train[column].unique())^ set(test[column].unique())
 if xor_values:
print('Column', column, 'has', len(xor_values), 'XOR values')
 train[column] = train[column].apply(replace_xor)
 test[column] = test[column].apply(replace_xor)
 else:
print('Column', column, 'has no XOR values' )
def multi_label_stratified_group_k_fold(label_arr: np.array, gid_arr: np.array, n_fold: int, seed: int=42):

 np.random.seed(seed)
 random.seed(seed)
 start_time = time.time()
n_train, n_class = label_arr.shape
gid_unique = sorted(set(gid_arr))
n_group = len(gid_unique)
 gid2aid = dict(zip(gid_unique, range(n_group)))
 aid_arr = np.vectorize(lambda x: gid2aid[x] )(gid_arr)
 cnts_by_class = label_arr.sum(axis=0)
 col, row = np.array(sorted(enumerate(aid_arr), key=lambda x: x[1])).T
cnts_by_group = coo_matrix(
(np.ones(len(label_arr)) ,(row, col))
).dot(coo_matrix(label_arr)).toarray().astype(int)
 del col
del row
cnts_by_fold = np.zeros(( n_fold, n_class), int)
 groups_by_fold = [[] for fid in range(n_fold)]
group_and_cnts = list(enumerate(cnts_by_group))
np.random.shuffle(group_and_cnts)
 print("finished preparation", time.time() - start_time)
 for aid, cnt_by_g in sorted(group_and_cnts, key=lambda x: -np.std(x[1])) :
best_fold = None
min_eval = None
for fid in range(n_fold):
cnts_by_fold[fid] += cnt_by_g
fold_eval =(cnts_by_fold / cnts_by_class ).std(axis=0 ).mean()
cnts_by_fold[fid] -= cnt_by_g
if min_eval is None or fold_eval < min_eval:
min_eval = fold_eval
best_fold = fid
cnts_by_fold[best_fold] += cnt_by_g
groups_by_fold[best_fold].append(aid)
 print("finished assignment.", time.time() - start_time)
 gc.collect()
idx_arr = np.arange(n_train)
 for fid in range(n_fold):
val_groups = groups_by_fold[fid]
val_indexs_bool = np.isin(aid_arr, val_groups)
 train_indexs = idx_arr[~val_indexs_bool]
val_indexs = idx_arr[val_indexs_bool]
print("[fold {}]".format(fid), end=" ")
 print("n_group:(train, val)=({}, {})".format(n_group - len(val_groups), len(val_groups)) , end=" ")
 print("n_sample:(train, val)=({}, {})".format(len(train_indexs), len(val_indexs)))
 yield train_indexs, val_indexs<concatenate>
y_train = train['target'].copy()
x_train = train.drop('target', axis=1)
 del train
x_test = test.copy()
del test
label_arr = train[CLASSES].values
group_id = train.PatientID.values
train_val_indexs = list(
 multi_label_stratified_group_k_fold(label_arr, group_id, N_FOLD, RANDAM_SEED))<feature_engineering>
train_part = len(x_train)
 traintest = pd.concat([x_train, x_test])
 df = pd.get_dummies(traintest, columns=traintest.columns, drop_first=True, sparse=True)
 x_train = df[:train_part]
x_test = df[train_part:]
del df
del traintest
train["fold"] = -1
for fold_id,(trn_idx, val_idx)in enumerate(train_val_indexs):
train.loc[val_idx, "fold"] = fold_id
train.groupby("fold")[CLASSES].sum()<train_model>
x_train = x_train.sparse.to_coo().tocsr()
x_test = x_test.sparse.to_coo().tocsr()
def resize_images(img_id, input_dir, output_dir, resize_to=(512, 512), ext="png"):
img_path = input_dir / f"{img_id}.jpg"
save_path = output_dir / f"{img_id}.{ext}"
img = cv2.imread(str(img_path), cv2.IMREAD_GRAYSCALE)
 img = cv2.resize(img, resize_to)
 cv2.imwrite(str(save_path), img,)
 TEST_RESIZED = TMP / "test_{0}x{1}".format(*IMAGE_SIZE)
 TEST_RESIZED.mkdir(exist_ok=True)
 TEST_RESIZED
_ = Parallel(n_jobs=2, verbose=5 )([
delayed(resize_images )(img_id, TEST, TEST_RESIZED, IMAGE_SIZE, "png")
 for img_id in smpl_sub.StudyInstanceUID.values
] )<choose_model_class>
from sklearn.model_selection import KFold, StratifiedKFold
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import roc_auc_score
from cross_validation_framework import *
def get_activation(activ_name: str="relu"):

 act_dict = {
"relu": nn.ReLU(inplace=True),
"tanh": nn.Tanh() ,
"sigmoid": nn.Sigmoid() ,
"identity": nn.Identity() }
if activ_name in act_dict:
return act_dict[activ_name]
else:
raise NotImplementedError
class Conv2dBNActiv(nn.Module):

 def __init__(
 self, in_channels: int, out_channels: int,
kernel_size: int, stride: int=1, padding: int=0,
bias: bool=False, use_bn: bool=True, activ: str="relu" 
):

 super(Conv2dBNActiv, self ).__init__()
layers = []
layers.append(nn.Conv2d(
 in_channels, out_channels,
kernel_size, stride, padding, bias=bias))
if use_bn:
layers.append(nn.BatchNorm2d(out_channels))
layers.append(get_activation(activ))
self.layers = nn.Sequential(*layers)
 def forward(self, x):

 return self.layers(x)
 class SSEBlock(nn.Module):

 def __init__(self, in_channels: int):

 super(SSEBlock, self ).__init__()
self.channel_squeeze = nn.Conv2d(
 in_channels=in_channels, out_channels=1,
kernel_size=1, stride=1, padding=0, bias=False)
 self.sigmoid = nn.Sigmoid()
def forward(self, x):

 h = self.sigmoid(self.channel_squeeze(x))
return x * h
class SpatialAttentionBlock(nn.Module):

 def __init__(
 self, in_channels: int,
out_channels_list: tp.List[int], 
):

 super(SpatialAttentionBlock, self ).__init__()
self.n_layers = len(out_channels_list)
 channels_list = [in_channels] + out_channels_list
assert self.n_layers > 0
assert channels_list[-1] == 1
for i in range(self.n_layers - 1):
in_chs, out_chs = channels_list[i: i + 2]
layer = Conv2dBNActiv(in_chs, out_chs, 3, 1, 1, activ="relu")
 setattr(self, f"conv{i + 1}", layer)
 in_chs, out_chs = channels_list[-2:]
layer = Conv2dBNActiv(in_chs, out_chs, 3, 1, 1, activ="sigmoid")
 setattr(self, f"conv{self.n_layers}", layer)
 def forward(self, x):

 h = x
for i in range(self.n_layers):
h = getattr(self, f"conv{i + 1}" )(h)
 h = h * x
return h<choose_model_class>
logit_param_grid = {
'C': [0.100, 0.150, 0.120, 0.125, 0.130, 0.135, 0.140, 0.145, 0.150]
}
logit_grid = GridSearchCV(LogisticRegression(solver='lbfgs'), logit_param_grid,
scoring='roc_auc', cv=5, n_jobs=-1, verbose=0)
 logit_grid.fit(x_train, y_train)
 best_C = logit_grid.best_params_['C']
print('Best C:', best_C )
class SingleHeadModel(nn.Module):
def __init__(
 self, base_name: str='resnext50_32x4d', out_dim: int=11, pretrained=False 
):

 self.base_name = base_name
super(SingleHeadModel, self ).__init__()
base_model = timm.create_model(base_name, pretrained=pretrained)
 in_features = base_model.num_features
base_model.reset_classifier(0)
 self.backbone = base_model
self.head_fc = nn.Sequential(
 nn.Linear(in_features, in_features),
nn.ReLU(inplace=True),
nn.Dropout(0.5),
nn.Linear(in_features, out_dim))
def forward(self, x):

 h = self.backbone(x)
 h = self.head_fc(h)
 return h
class MultiHeadModel(nn.Module):
def __init__(
 self, base_name: str='resnext50_32x4d',
out_dims_head: tp.List[int]=[3, 4, 3, 1], pretrained=False):

 self.base_name = base_name
self.n_heads = len(out_dims_head)
 super(MultiHeadModel, self ).__init__()
base_model = timm.create_model(base_name, pretrained=pretrained)
 in_features = base_model.num_features
base_model.reset_classifier(0, '')
 self.backbone = base_model
for i, out_dim in enumerate(out_dims_head):
layer_name = f"head_{i}"
layer = nn.Sequential(
 SpatialAttentionBlock(in_features, [64, 32, 16, 1]),
nn.AdaptiveAvgPool2d(output_size=1),
nn.Flatten(start_dim=1),
nn.Linear(in_features, in_features),
nn.ReLU(inplace=True),
nn.Dropout(0.5),
nn.Linear(in_features, out_dim))
setattr(self, layer_name, layer)
 def forward(self, x):

 h = self.backbone(x)
 hs = [
getattr(self, f"head_{i}" )(h)for i in range(self.n_heads)]
y = torch.cat(hs, axis=1)
 return y 
<categorify>
logit = LogisticRegression(C=best_C, solver='lbfgs', class_weight='balanced', max_iter=10000)
 cv = KFold(n_splits=10, random_state=42)
 oof, trained_estimators = fit(ScikitLearnPredictProbaEstimator(logit), roc_auc_score, x_train, y_train, cv)
 y = predict(trained_estimators, x_test )
class LabeledImageDataset(data.Dataset):

 def __init__(
 self,
file_list: tp.List[
tp.Tuple[tp.Union[str, Path], tp.Union[int, float, np.ndarray]]],
transform_list: tp.List[tp.Dict], 
):

 self.file_list = file_list
self.transform = ImageTransformForCls(transform_list)
 def __len__(self):

 return len(self.file_list)
 def __getitem__(self, index):

 img_path, label = self.file_list[index]
img = self._read_image_as_array(img_path)
 img, label = self.transform(( img, label))
return img, label
def _read_image_as_array(self, path: str):

 img_arr = cv2.imread(str(path))
img_arr = cv2.cvtColor(img_arr, cv2.COLOR_BGR2RGB)
 return img_arr<create_dataframe>
submission = pd.read_csv('.. /input/cat-in-the-dat/sample_submission.csv', index_col='id')
 submission['target'] = y
submission.to_csv('logit.csv' )