Kaggle competition: Categorical Feature Encoding Challenge II
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge-ii<categorify>
os.mkdir("./working")
 os.environ['USER'] = 'root'
os.system('pip install.. /input/xlearn/xlearn/xlearn-0.40a1/')
class CassavaDataset(Dataset):
def __init__(
 self, df, data_root, transforms=None, output_label=True 
):
super().__init__()
self.df = df.reset_index(drop=True ).copy()
self.transforms = transforms
self.data_root = data_root
self.output_label = output_label
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index: int):
if self.output_label:
target = self.df.iloc[index]['label']
path = "{}/{}".format(self.data_root, self.df.iloc[index]['image_id'])
 img = get_img(path)
 if self.transforms:
img = self.transforms(image=img)['image']
if self.output_label == True:
return img, target
else:
return img<import_modules>
warnings.filterwarnings('ignore')
 pd.options.display.max_columns = 50
BIN_COL = [f'bin_{i}' for i in range(5)]
NOM_COL = [f'nom_{i}' for i in range(10)]
ORD_COL = [f'ord_{i}' for i in range(6)]
NOM_5_9 = ['nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']
NOM_0_4 = ['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4']
DATE_COL = ['day','month']
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
for dirname, _, filenames in os.walk('/kaggle/working/libffm-binaries'):
for filename in filenames:
print(os.path.join(dirname, filename))
HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize 
)
 HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize 
)
 def get_train_transforms() :
return Compose([
RandomResizedCrop(CFG['img_size'], CFG['img_size']),
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
ShiftScaleRotate(p=0.5),
HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),
RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
CoarseDropout(p=0.5),
Cutout(p=0.5),
ToTensorV2(p=1.0),
], p=1.)
def get_valid_transforms() :
return Compose([
Resize(CFG['img_size'], CFG['img_size'], p=1.) ,
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.)
def get_inference_transforms() :
return Compose([
Resize(CFG['img_size'], CFG['img_size']),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.)<choose_model_class>
%%time
def read_csv() :
train = pd.read_csv('.. /input/cat-in-the-dat-ii/train.csv')
 test = pd.read_csv('.. /input/cat-in-the-dat-ii/test.csv')
 train_id = train['id']
test_id = test['id']
train.drop('id', axis=1, inplace=True)
 test.drop('id', axis=1, inplace=True)
 return train, test, train_id, test_id
train, test, train_id, test_id = read_csv()
def preprocessing(df):
df['bin_missing'] =(df[['bin_0', 'bin_1', 'bin_2', 'bin_4']].isnull().sum(axis=1)>0 ).replace({True:1, False:0})
 df['nom_missing'] =(df[NOM_COL].isnull().sum(axis=1)>0 ).replace({True:1, False:0})
 df['ord_missing'] =(df[ORD_COL].isnull().sum(axis=1)>0 ).replace({True:1, False:0})
 df['date_missing'] =(df[DATE_COL].isnull().sum(axis=1)>0 ).replace({True:1, False:0})
 df.day = df.day.replace({3:5,2:6,1:7})
 df.loc[df.ord_5.notnull() , 'ord_5_1'] = df.loc[df.ord_5.notnull() , 'ord_5'].apply(lambda x: x[0])
 df.loc[df.ord_5.notnull() , 'ord_5_2'] = df.loc[df.ord_5.notnull() , 'ord_5'].apply(lambda x: x[1])
 return df
train, test, train_id, test_id = read_csv()
train = preprocessing(train)
 test = preprocessing(test)
 print(f'train day unique value:{train.day.unique() }')
 print(f'test day unique value:{test.day.unique() }')
 for col in test.columns:
if len(set(train[col].dropna().unique().tolist())^ set(test[col].dropna().unique().tolist())) >0:
train_only = list(set(train[col].dropna().unique().tolist())- set(test[col].dropna().unique().tolist()))
test_only = list(set(test[col].dropna().unique().tolist())- set(train[col].dropna().unique().tolist()))
print(col, '(train only)', train_only, '(test only)', test_only)
 train.loc[train[col].isin(train_only), col] = np.NaN
test.loc[test[col].isin(test_only), col] = np.NaN
test.insert(1, 'target', 0)
 drop_cols = ['bin_3', 'ord_5']
train.drop(columns=drop_cols, inplace=True)
 test.drop(columns=drop_cols, inplace=True )
class CassvaImgClassifier(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):
super().__init__()
self.model = timm.create_model(model_arch, pretrained=pretrained)
 n_features = self.model.classifier.in_features
self.model.classifier = nn.Linear(n_features, n_class)
 def forward(self, x):
x = self.model(x)
 return x<feature_engineering>
%%time
features = [_f for _f in train if _f not in ['id', 'target']]
def factor_encoding(train, test):
assert sorted(train.columns)== sorted(test.columns)
 full = pd.concat([train, test], axis=0, sort=False)
 for f in full:
full[f], _ = pd.factorize(full[f])
 full[f] += 1
return full.iloc[:len(train)], full.iloc[len(train):]
train_f, test_f = factor_encoding(train[features], test[features])
 print(train_f,'-'*20)
 print(train_f.head(10))
print(test_f,'-'*20)
 print(test_f.head(10))
class LibFFMEncoder(object):
def __init__(self):
self.encoder = 1
self.encoding = {}
def encode_for_libffm(self, row):
txt = f"{row[0]}"
for i, r in enumerate(row[1:]):
try:
txt += f' {i+1}:{self.encoding[(i, r)]}:1'
except KeyError:
self.encoding[(i, r)] = self.encoder
self.encoder += 1
txt += f' {i+1}:{self.encoding[(i, r)]}:1'
return txt
SPLITS = 5
fold_ids = [
[trn_, val_] for(trn_, val_)in KFold(SPLITS,True,1 ).split(train)
 ]
for fold_,(trn_, val_)in enumerate(fold_ids):
encoder = LibFFMEncoder()
libffm_format_trn = pd.concat([train['target'].iloc[trn_], train_f.iloc[trn_]], axis=1 ).apply(
 lambda row: encoder.encode_for_libffm(row), raw=True, axis=1 
)
 libffm_format_val = pd.concat([train['target'].iloc[val_], train_f.iloc[val_]], axis=1 ).apply(
 lambda row: encoder.encode_for_libffm(row), raw=True, axis=1 
)
 print(train['target'].iloc[trn_].shape, train['target'].iloc[val_].shape, libffm_format_val.shape)
 libffm_format_trn.to_csv(f'libffm_trn_fold_{fold_+1}.txt', index=False, header=False)
 libffm_format_val.to_csv(f'libffm_val_fold_{fold_+1}.txt', index=False, header=False)
 encoder = LibFFMEncoder()
libffm_format_trn = pd.concat([train['target'], train_f], axis=1 ).apply(
 lambda row: encoder.encode_for_libffm(row), raw=True, axis=1 
)
 libffm_format_tst = pd.concat([test['target'], test_f], axis=1 ).apply(
 lambda row: encoder.encode_for_libffm(row), raw=True, axis=1 
)
 libffm_format_trn.to_csv(f'libffm_trn.txt', index=False, header=False)
 libffm_format_tst.to_csv(f'libffm_tst.txt', index=False, header=False )
if __name__ == '__main__':
seed_everything(CFG['seed'])
 folds = StratifiedKFold(n_splits=CFG['fold_num'] ).split(np.arange(train.shape[0]), train.label.values)
 for fold,(trn_idx, val_idx)in enumerate(folds):
if fold > 0:
break
print('Inference fold {} started'.format(fold))
test = pd.DataFrame()
test['image_id'] = list(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
test_ds = CassavaDataset(test, '.. /input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms() , output_label=False)
 tst_loader = torch.utils.data.DataLoader(
 test_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 device = torch.device(CFG['device'])
 model = CassvaImgClassifier(CFG['model_arch'], train.label.nunique() ).to(device)
 tst_preds = []
for i, epoch in enumerate(CFG['used_epochs']):
model.load_state_dict(torch.load('.. /input/cutmix/{}_fold_{}_{}'.format(CFG['model_arch'], fold, epoch)))
 with torch.no_grad() :
for _ in range(CFG['tta']):
tst_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, tst_loader, device)]
tst_preds = np.mean(tst_preds, axis=0)
 del model
torch.cuda.empty_cache()<feature_engineering>
%%time
ffm_model = xl.create_ffm()
outputs = []
param = {'task':'binary',
'lr':0.2,
'k':4,
'lambda':0.0002,
'metric':'auc',
'epoch': 15
}
for fold_ in range(1, SPLITS+1):
print(f'fold: {fold_}')
 model = f"libffm_fold_{fold_}_model"
trn_fold_txt = f"libffm_trn_fold_{fold_}.txt"
val_fold_txt = f"libffm_val_fold_{fold_}.txt"
val_preds_fold_txt = f"val_preds_fold_{fold_}.txt"
ffm_model.setTrain(trn_fold_txt)
 ffm_model.setValidate(val_fold_txt)
 print(' fitting...')
 ffm_model.fit(param, 'model.output')
 print(' make predictions...')
 ffm_model.setTest(val_fold_txt)
 ffm_model.setSigmoid()
ffm_model.predict("model.output", val_preds_fold_txt)
 gc.collect()
print(' auc score:',
roc_auc_score(
 train['target'].iloc[fold_ids[fold_-1][1]],
pd.read_csv(val_preds_fold_txt, header=None ).values[:,0]))
test['label'] = np.argmax(tst_preds, axis=1)
 test.head()<save_to_csv>
gc.collect()
test.to_csv('submission.csv', index=False )<install_modules>
oof_preds = np.zeros(train.shape[0])
 for fold_,(_, val_)in enumerate(fold_ids):
oof_preds[val_] = pd.read_csv(f'val_preds_fold_{fold_+1}.txt', header=None ).values[:, 0]
print(roc_auc_score(train['target'], oof_preds))
!pip install --quiet /kaggle/input/kerasapplications
!pip install --quiet /kaggle/input/efficientnet-git<set_options>
%%time
ffm_model = xl.create_ffm()
ffm_model.setTrain("libffm_trn.txt")
 ffm_model.fit(param, 'model.output' )
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
seed = 0
seed_everything(seed)
 warnings.filterwarnings('ignore' )<set_options>
%%time
ffm_model.setTest('libffm_tst.txt')
 ffm_model.setSigmoid()
ffm_model.predict('model.output', 'tst_preds.txt' )
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print(f'Running on TPU {tpu.master() }')
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )<define_variables>
gc.collect()
BATCH_SIZE = 16 * REPLICAS
HEIGHT = 512
WIDTH = 512
CHANNELS = 3
N_CLASSES = 5
TTA_STEPS = 8<define_search_space>
submission = pd.DataFrame()
submission['id'] = test_id
submission['target'] = pd.read_csv('tst_preds.txt', header=None ).values[:,0]
submission.to_csv('xlearn_prediction.csv', index=False )