Kaggle competition: Categorical Feature Encoding Challenge II
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge-ii<load_from_csv>
!pip install --upgrade xgbfir
test_csv = df.copy()
test_csv['image_id'] = PATH + test_csv['image_id']
test_ds = DiseaseDatasetInference(test_csv, transform=get_inference_transforms() , opt_label=False)
 test_loader = torch.utils.data.DataLoader(test_ds, batch_size=CFG['valid_bs'], shuffle=False, pin_memory=False )<predict_on_test>
import numpy as np
import pandas as pd
import sklearn as sk
import matplotlib as mpl
import matplotlib.pyplot as plt
import seaborn as sbn
import sys
from datetime import datetime
import category_encoders as ce
from scipy.stats import chi2_contingency
from statsmodels.stats.multitest import fdrcorrection, multipletests
from sklearn.model_selection import train_test_split, StratifiedKFold
from sklearn.metrics import roc_auc_score, roc_curve, recall_score, accuracy_score, precision_recall_curve
from sklearn.preprocessing import OneHotEncoder, StandardScaler, LabelEncoder
import xgboost as xgb
import xgbfir
import string
import joblib
from category_encoders import TargetEncoder, OrdinalEncoder
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import CategoricalNB
def inference(model, data_loader, device):
preds = []
model.eval()
test_tqdm = tq.tqdm(data_loader, total=len(data_loader), desc="Testing", position=0, leave=True)
 for images in test_tqdm:
images = images.to(device)
 preds.extend(model(images ).detach().cpu().numpy())
 return preds<choose_model_class>
print("python", sys.version)
 for module in np, pd, mpl, sbn, ce, sk, xgb, joblib:
print(module.__name__, module.__version__ )
class CassavaImageClassifier(nn.Module):
def __init__(self, efnet_arch, n_class, pretrained=False):
super().__init__()
self.efnet_model = timm.create_model(efnet_arch, pretrained=pretrained)
 efnet_features = self.efnet_model.classifier.in_features
self.efnet_model.classifier = nn.Linear(efnet_features, n_class)
 def forward(self, x):
efnet_opt = self.efnet_model(x)
 return efnet_opt<load_pretrained>
np.random.seed(0 )
effnet_preds = []
for effnet_model_name in CFG['effnet_models']:
print("Model: ", effnet_model_name)
 effnet_model = torch.load('/kaggle/input/effete-cassava/ensemble/'+effnet_model_name, map_location=torch.device(CFG['device']))
with torch.no_grad() :
for i in range(CFG['tta']):
effnet_preds += [inference(effnet_model, test_loader, CFG['device'])]
effnet_preds = np.mean(effnet_preds, axis=0 )<concatenate>
df = pd.read_csv('.. /input/cat-in-the-dat-ii/train.csv' ).drop('id',axis=1)
 test = pd.read_csv(r'.. /input/cat-in-the-dat-ii/test.csv', index_col=0)
 sample_submission = pd.read_csv(r'.. /input/cat-in-the-dat-ii/sample_submission.csv', index_col=0 )
effnet_outcomes = pd.concat([df['image_id'], pd.DataFrame(effnet_preds)], axis=1 ).sort_values(['image_id'] )<choose_model_class>
( df.isna().sum(axis=1)==0 ).sum() /df.shape[0]
class CassavaImageClassifier(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):
super().__init__()
self.model = timm.create_model(model_arch, pretrained=pretrained)
 n_features = self.model.fc.in_features
self.model.fc = nn.Linear(n_features, n_class)
 def forward(self, x):
x = self.model(x)
 return x<load_pretrained>
( test.isna().sum(axis=1)==0 ).sum() /test.shape[0]
resnet_preds = []
for resnet_model_name in CFG['resnet_models']:
print("Model: ", resnet_model_name)
 resnet_model = torch.load('/kaggle/input/resnet-cassava/resnet/'+resnet_model_name, map_location=torch.device(CFG['device']))
with torch.no_grad() :
for i in range(CFG['tta']):
resnet_preds += [inference(resnet_model, test_loader, CFG['device'])]
resnet_preds = np.mean(resnet_preds, axis=0 )<concatenate>
df.isna().sum(axis=1 ).value_counts()
resnet_outcomes = pd.concat([df['image_id'], pd.DataFrame(resnet_preds)], axis=1 ).sort_values(['image_id'] )<categorify>
def cochran_criterion(crosstab):
criterion = True
E = []
N = df.shape[0]
for O_i in crosstab.sum(axis=1):
for O_j in crosstab.sum() :
if O_i*O_j/N==0:
critetion = False
E.append(O_i*O_j/N > 4)
 criterion = criterion &(np.mean(E)>0.8)
 return criterion
final_preds =(effnet_outcomes.drop('image_id', axis=1)*0.5 + resnet_outcomes.drop('image_id', axis=1)*0.5 ).to_numpy()
final_preds = softmax(final_preds ).argmax(1 )<compute_test_metric>
def chi2_test(X,Y):
crosstab = pd.crosstab(X, Y)
 criterion = cochran_criterion(crosstab)
 chi2, p = chi2_contingency(crosstab)[:2]
return [criterion, chi2, p]
accuracy_score(final_preds, df['label'].values )<save_to_csv>
start = datetime.now()
df_total = pd.concat([df.drop('target', axis=1), test])
 chi2_missingness = pd.DataFrame(columns=['Cochran_criterion', 'Chi2', 'p_value'])
 for col1 in df_total.columns:
for col2 in df_total.columns.drop(col1):
missingness_indicator = df_total[col1].isna()
other_variable = df_total[col2]
chi2_missingness.loc[col1 + '_' + col2] = chi2_test(missingness_indicator, other_variable)
 for col in df.columns.drop('target'):
missingness_indicator = df[col].isna()
chi2_missingness.loc[col + '_target'] = chi2_test(missingness_indicator, df.target)
 runtime = datetime.now() - start
print('Runtime ', runtime )
submit = pd.DataFrame({'image_id': df['image_id'].values, 'label': final_preds})
 submit.to_csv('submission.csv', index=False )<install_modules>
chi2_missingness.sort_values('p_value', inplace=True)
 reject, pvalue_corrected = fdrcorrection(pvals=chi2_missingness.p_value, alpha=0.05, method='indep', is_sorted=True)
 chi2_missingness['p_value_corrected'] = pvalue_corrected
chi2_missingness['BH_reject'] = chi2_missingness['Cochran_criterion'] & reject
chi2_missingness.to_csv('chi2_tests_missingness' )
!pip install --quiet /kaggle/input/kerasapplications
!pip install --quiet /kaggle/input/efficientnet-git<set_options>
significant_tests = chi2_missingness[chi2_missingness['BH_reject']==True]
print('Number of statistically significant tests: ', significant_tests.shape[0] )
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
seed = 0
seed_everything(seed)
 warnings.filterwarnings('ignore' )<set_options>
def predict_missingness(col):
X = df_total.drop(col, axis=1)
 y = df_total[col].isna()
cols = X.columns.tolist()
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, stratify=y, random_state=0)
 X_train.reset_index(drop=True, inplace=True)
 y_train.reset_index(drop=True, inplace=True)
 X_train_te = X_train.copy()
for index_fit, index_transform in StratifiedKFold(n_splits=5, random_state=0, shuffle=True ).split(X_train, y_train):
target_enc = TargetEncoder(cols = cols, smoothing=0.5, min_samples_leaf=5)
 target_enc.fit(X_train.iloc[index_fit,:], y_train.iloc[index_fit])
 X_train_te.loc[index_transform,:] = target_enc.transform(X_train.iloc[index_transform, :])
 X_train_te = X_train_te.astype(float)
 target_enc = TargetEncoder(cols = cols, smoothing=0.5, min_samples_leaf=5)
 target_enc.fit(X_train, y_train)
 X_val_te = target_enc.transform(X_val)
 weight = sum(y_train==0)/sum(y_train==1)
 gbt = xgb.XGBClassifier(n_estimators=3000, random_state=0, objective='binary:logistic', scale_pos_weight=weight,
learning_rate=0.2, max_depth=4, subsample=0.8, colsample_bytree = 0.8, min_child_weight=1000)
 gbt.fit(X_train_te, y_train, eval_set= [(X_train_te, y_train),(X_val_te, y_val)], eval_metric=['auc'], early_stopping_rounds=50,
verbose=False)
 y_pred = gbt.predict_proba(X_train_te)
 train_score = roc_auc_score(y_train, y_pred[:,1])
 y_pred = gbt.predict_proba(X_val_te)
 val_score = roc_auc_score(y_val, y_pred[:,1])
 return(train_score, val_score )
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print(f'Running on TPU {tpu.master() }')
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )<define_variables>
start = datetime.now()
pred_missingness = pd.DataFrame(columns=['Feature', 'train_score', 'val_score'])
 for col in df_total.columns:
train_score, val_score = predict_missingness(col)
 pred_missingness.loc[col] = [col, train_score, val_score]
pred_missingness.sort_values('val_score', ascending=False, inplace=True)
 pred_missingness.to_csv('pred_missingness.csv', index=False)
 runtime = datetime.now() - start
print('Runtime ', runtime )
BATCH_SIZE = 16 * REPLICAS
HEIGHT = 512
WIDTH = 512
CHANNELS = 3
N_CLASSES = 5
TTA_STEPS = 0
USE_REGULAR = False
USE_SCL = True<define_search_space>
def predict_feature(col):
df_new = df_total[df_total[col].isna() ==False].astype(str)
 X = df_new.drop(col, axis=1)
 y = df_new[col]
le = LabelEncoder()
y = le.fit_transform(y)
 train_scores = []
val_scores = []
for index_train, index_val in StratifiedKFold(n_splits=5, random_state=42, shuffle=True ).split(X, y):
X_train = X.iloc[index_train]
y_train = y[index_train]
X_valid = X.iloc[index_val]
y_valid = y[index_val]
oe = OrdinalEncoder()
X_train_oe = oe.fit_transform(X_train.astype(str))
X_val_oe = oe.transform(X_valid.astype(str))
X_train_oe += 2
X_val_oe += 2
nb = CategoricalNB()
nb.fit(X_train_oe, y_train)
 if np.max(y)==1:
y_pred = nb.predict_proba(X_train_oe)
 train_score = roc_auc_score(y_train, y_pred[:,1])
 train_scores.append(train_score)
 y_pred = nb.predict_proba(X_val_oe)
 val_score = roc_auc_score(y_valid, y_pred[:,1])
 val_scores.append(val_score)
 else:
y_pred = nb.predict_proba(X_train_oe)
 train_score = roc_auc_score(y_train, y_pred, multi_class='ovr')
 train_scores.append(train_score)
 y_pred = nb.predict_proba(X_val_oe)
 val_score = roc_auc_score(y_valid, y_pred, multi_class='ovr')
 val_scores.append(val_score)
 return(train_scores, val_scores )
def data_augment(image, label):
p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_pixel_1 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_pixel_2 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_pixel_3 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 if p_spatial >.75:
image = tf.image.transpose(image)
 if p_rotate >.75:
image = tf.image.rot90(image, k=3)
 elif p_rotate >.5:
image = tf.image.rot90(image, k=2)
 elif p_rotate >.25:
image = tf.image.rot90(image, k=1)
 if p_pixel_1 >=.4:
image = tf.image.random_saturation(image, lower=.7, upper=1.3)
 if p_pixel_2 >=.4:
image = tf.image.random_contrast(image, lower=.8, upper=1.2)
 if p_pixel_3 >=.4:
image = tf.image.random_brightness(image, max_delta=.1)
 if p_crop >.6:
if p_crop >.9:
image = tf.image.central_crop(image, central_fraction=.5)
 elif p_crop >.8:
image = tf.image.central_crop(image, central_fraction=.6)
 elif p_crop >.7:
image = tf.image.central_crop(image, central_fraction=.7)
 else:
image = tf.image.central_crop(image, central_fraction=.8)
 elif p_crop >.3:
crop_size = tf.random.uniform([], int(HEIGHT*.6), HEIGHT, dtype=tf.int32)
 image = tf.image.random_crop(image, size=[crop_size, crop_size, CHANNELS])
 image = tf.image.resize(image, size=[HEIGHT, WIDTH])
 return image, label<load_from_csv>
start = datetime.now()
pred_features = pd.DataFrame(columns=['Feature', 'train_score', 'val_score', 'std_val_score'])
 high_cardinality_features = ['nom_' + str(i)for i in range(5,10)] + ['ord_5']
for col in df_total.columns.drop(high_cardinality_features):
train_score, val_score = predict_feature(col)
 pred_features.loc[col] = [col, np.mean(train_score), np.mean(val_score), np.std(val_score)]
pred_features.sort_values('val_score', ascending=False, inplace=True)
 pred_features.to_csv('pred_features.csv', index=False)
 runtime = datetime.now() - start
print('Runtime ', runtime )
database_base_path = '/kaggle/input/cassava-leaf-disease-classification/'
submission = pd.read_csv(f'{database_base_path}sample_submission.csv')
 display(submission.head())
 TEST_FILENAMES = tf.io.gfile.glob(f'{database_base_path}test_tfrecords/ld_test*.tfrec')
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 print(f'GCS: test: {NUM_TEST_IMAGES}' )<train_model>
df.bin_4.fillna('NaN' ).value_counts()
model_path_list = glob.glob('/kaggle/input/cassava-leaf-supervised-contrastive-learning/model_reg*.h5')
 model_path_list.sort()
print('Models to predict:')
 print(*model_path_list, sep='
')
 model_path_list_scl = glob.glob('/kaggle/input/cassava-leaf-supervised-contrastive-learning/model_scl*.h5')
 model_path_list_scl.sort()
print('
Models to predict:')
 print(*model_path_list_scl, sep='
' )<categorify>
df.nom_5.fillna('NaN' ).value_counts()
def encoder_fn(input_shape):
inputs = L.Input(shape=input_shape, name='inputs')
 base_model = efn.EfficientNetB3(input_tensor=inputs,
include_top=False,
weights=None,
pooling='avg')
 model = Model(inputs=inputs, outputs=base_model.outputs)
 return model
def classifier_fn(input_shape, N_CLASSES, encoder, trainable=True):
for layer in encoder.layers:
layer.trainable = trainable
inputs = L.Input(shape=input_shape, name='inputs')
 features = encoder(inputs)
 features = L.Dropout (.5 )(features)
 features = L.Dense(1000, activation='relu' )(features)
 features = L.Dropout (.5 )(features)
 outputs = L.Dense(N_CLASSES, activation='softmax', name='outputs', dtype='float32' )(features)
 model = Model(inputs=inputs, outputs=outputs)
 return model<categorify>
def preprocessing_1(df_train, df_test):

 y_train = df_train['target'].copy()
df = pd.concat([df_train.drop('target', axis=1), df_test])
 df = df.fillna(value={'bin_0': df.bin_0.mode() [0], 'bin_1':df.bin_1.mode() [0], 'bin_2':df.bin_2.mode() [0], 'bin_3':df.bin_3.mode() [0],
'bin_4':df.bin_4.mode() [0], 'nom_0':df.nom_0.mode() [0], 'nom_1':'nan', 'nom_2':'nan', 'nom_3':'nan', 'nom_4':'nan',
'nom_5':'nan', 'nom_6':'nan', 'nom_7':'nan', 'nom_8':'nan', 'nom_9':'nan', 'ord_0':df.ord_0.median() , 'ord_5':'0',
'day':8, 'month':13})
 df['ord_5'] = df['ord_5'].astype(str ).str[0]
replace_letters = {j:i for i,j in enumerate(list(string.ascii_uppercase + string.ascii_lowercase)) }
replace_letters['0'] = np.nan
df = df.replace({'bin_3':{'F':0, 'T':1}, 'bin_4':{'N':0, 'Y':1},'ord_1': {'Novice':1, 'Contributor':2, 'Expert':3, 'Master':4, 'Grandmaster':5},
'ord_2':{'Freezing': 1, 'Cold':2, 'Warm':3, 'Hot':4, 'Boiling Hot': 5, 'Lava Hot':6},'ord_3': replace_letters, 'ord_4':replace_letters,
'ord_5':replace_letters})
 df = df.fillna(value={'ord_1':df['ord_1'].median() , 'ord_2':df['ord_2'].median() , 'ord_3':df['ord_3'].median() , 'ord_4':df['ord_4'].median() , 'ord_5':df['ord_5'].median() })
 one_hot_cols = ['nom_0','nom_1','nom_2','nom_3','nom_4']
X_cat = df[one_hot_cols].astype(str ).copy()
one_hot_enc = OneHotEncoder()
X_cat_1hot = one_hot_enc.fit_transform(X_cat)
 X_cat_1hot = pd.DataFrame(X_cat_1hot.toarray() , index=X_cat.index, columns = one_hot_enc.get_feature_names())
 X_cat_1hot.drop(['x'+str(i)+'_' + X_cat['nom_' + str(i)].mode() [0] for i in range(5)], axis=1, inplace=True)
 X = pd.concat([df.drop(one_hot_cols, axis=1), X_cat_1hot], axis=1)
 X['sin_day'] = np.sin(2*np.pi*X.day/8)
 X['cos_day'] = np.cos(2*np.pi*X.day/8)
 X['sin_month'] = np.sin(2*np.pi*X.month/13)
 X['cos_month'] = np.cos(2*np.pi*X.month/13)
 X.drop(['day', 'month'], axis=1, inplace=True)
 X_train = X.iloc[:600000]
X_test = X.iloc[600000:1000000]
target_enc_cols = ['nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']
X_train_encoded = X_train.copy()
for index_fit, index_transform in StratifiedKFold(n_splits=5, random_state=42, shuffle=True ).split(X_train, y_train):
target_enc = TargetEncoder(cols = target_enc_cols, smoothing=0.5)
 target_enc.fit(X_train.iloc[index_fit,:], y_train.iloc[index_fit])
 X_train_encoded.loc[index_transform,:] = target_enc.transform(X_train.iloc[index_transform, :])
 target_enc = TargetEncoder(cols = target_enc_cols, smoothing=0.5)
 target_enc.fit(X_train, y_train)
 X_test_encoded = target_enc.transform(X_test)
 X_train_encoded.loc[:,target_enc_cols] = X_train_encoded[target_enc_cols].astype(float)
 X_train_encoded.loc[:, X_train_encoded.columns.drop(target_enc_cols ).to_list() ] = X_train_encoded.loc[:, X_train_encoded.columns.drop(target_enc_cols ).to_list() ].astype(float)
 X_test_encoded.loc[:,'ord_3'] = X_test_encoded.loc[:,'ord_3'].astype(float)
 X_test_encoded.loc[:,'ord_4'] = X_test_encoded.loc[:,'ord_4'].astype(float)
 return(X_train_encoded, y_train, X_test_encoded )
files_path = f'{database_base_path}test_images/'
test_size = len(os.listdir(files_path))
test_preds = np.zeros(( test_size, N_CLASSES))
if USE_REGULAR:
print('Inference for regular trainining models')
 with strategy.scope() :
encoder = encoder_fn(( None, None, CHANNELS))
model = classifier_fn(( None, None, CHANNELS), N_CLASSES, encoder)
 for model_path in model_path_list:
print(model_path)
 K.clear_session()
model.load_weights(model_path)
 if TTA_STEPS > 0:
test_ds = get_dataset(files_path, tta=True ).repeat()
ct_steps = TTA_STEPS *(( test_size/BATCH_SIZE)+ 1)
 preds = model.predict(test_ds, steps=ct_steps, verbose=1)[:(test_size * TTA_STEPS)]
preds = np.mean(preds.reshape(test_size, TTA_STEPS, N_CLASSES, order='F'), axis=1)
 test_preds += preds / len(model_path_list)
 else:
test_ds = get_dataset(files_path, tta=False)
 x_test = test_ds.map(lambda image, image_name: image)
 test_preds += model.predict(x_test)/ len(model_path_list)
 if USE_SCL:
print('
Inference for "Supervised Contrastive Learning" models')
 with strategy.scope() :
encoder = encoder_fn(( None, None, CHANNELS))
model = classifier_fn(( None, None, CHANNELS), N_CLASSES, encoder, trainable=False)
 for model_path in model_path_list_scl:
print(model_path)
 K.clear_session()
model.load_weights(model_path)
 if TTA_STEPS > 0:
test_ds = get_dataset(files_path, tta=True ).repeat()
ct_steps = TTA_STEPS *(( test_size/BATCH_SIZE)+ 1)
 preds = model.predict(test_ds, steps=ct_steps, verbose=1)[:(test_size * TTA_STEPS)]
preds = np.mean(preds.reshape(test_size, TTA_STEPS, N_CLASSES, order='F'), axis=1)
 test_preds += preds / len(model_path_list_scl)
 else:
test_ds = get_dataset(files_path, tta=False)
 x_test = test_ds.map(lambda image, image_name: image)
 test_preds += model.predict(x_test)/ len(model_path_list_scl)
 test_preds = np.argmax(test_preds, axis=-1)
 test_names_ds = get_dataset(files_path)
 image_names = [img_name.numpy().decode('utf-8')for img, img_name in iter(test_names_ds.unbatch())]<save_to_csv>
start = datetime.now()
X, y, X_test = preprocessing_1(df, test)
 end = datetime.now()
print('Duration: ', end-start )
submission = pd.DataFrame({'image_id': image_names, 'label': test_preds})
 submission.to_csv('submission.csv', index=False)
 display(submission.head() )<define_variables>
X.to_csv('X_1st_strategy.csv', index=False)
 y.to_csv('y.csv', index=False, header=True)
 X_test.to_csv('X_test_1st_strategy.csv', index=False )
package_path = '.. /input/pytorch-image-models/pytorch-image-models-master'
sys.path.append(package_path)
 DATA_DIR = '.. /input/cassava-leaf-disease-classification'
MODEL_DIR = '.. /input/efficientnet-baseline-train-amp-aug'<import_modules>
def display_scores(train_scores, val_scores, y_validations, y_predictions):
print('Training scores: ', train_scores)
 print('Mean training score: ', np.mean(train_scores))
print('Standard deviation of the training scores: ', np.std(train_scores), '
')
 print('Validation scores: ', val_scores)
 print('Mean validation score: ', np.mean(val_scores))
print('Standard deviation of the validation scores: ', np.std(val_scores), '
')
 y_valid = np.concatenate(tuple(y_validations))
y_pred = np.concatenate(tuple(y_predictions))
plot_precision_recall_vs_threshold(y_valid, y_pred)
 print('
')
 fpr, tpr, threshold = roc_curve(y_valid, y_pred)
 plot_roc_curve(fpr,tpr )
HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize 
)
<init_hyperparams>
start = datetime.now()
train_scores = []
val_scores = []
y_validations = []
y_predictions = []
for index_train, index_val in StratifiedKFold(n_splits=5, random_state=42, shuffle=True ).split(X, y):
X_train = X.iloc[index_train]
y_train = y.iloc[index_train]
X_valid = X.iloc[index_val]
y_valid = y.iloc[index_val]
weights = sum(y_train.values==0)/sum(y_train.values==1)
 xgb_gbrf = xgb.XGBClassifier(n_estimators=3000, random_state=0, objective='binary:logistic', scale_pos_weight=weights,
learning_rate=0.15, max_depth=2, subsample=0.7, min_child_weight=500,
colsample_bytree = 0.2, reg_lambda = 3.5, reg_alpha=1.5, num_parallel_tree= 5)
 xgb_gbrf.fit(X_train, y_train, eval_set= [(X_train, y_train),(X_valid, y_valid)], eval_metric=['auc'],
early_stopping_rounds=100, verbose=False)
 y_pred = xgb_gbrf.predict_proba(X_train)
 train_score = roc_auc_score(y_train, y_pred[:,1])
 train_scores.append(train_score)
 y_pred = xgb_gbrf.predict_proba(X_valid)
 val_score = roc_auc_score(y_valid, y_pred[:,1])
 val_scores.append(val_score)
 y_validations.append(y_valid)
 y_predictions.append(y_pred[:,1])
 display_scores(train_scores, val_scores, y_validations, y_predictions)
 runtime = datetime.now() - start
print('
Runtime ', runtime )
CFG = {
'fold_num': 5,
'seed': 719,
'model_arch': 'tf_efficientnet_b4_ns',
'img_size': 512,
'epochs': 10,
'train_bs': 32,
'valid_bs': 32,
'lr': 1e-4,
'num_workers': 4,
'accum_iter': 1,
'verbose_step': 1,
'device': 'cuda:0',
'tta': 3,
'used_epochs': [6,7,8,9],
'weights': [1,1,1,1]
}<load_from_csv>
gbrf_feature_importances = pd.DataFrame({'Predictors': X.columns, 'Normalized gains': xgb_gbrf.feature_importances_})
 gbrf_feature_importances.sort_values('Normalized gains',ascending=False, inplace=True)
 gbrf_feature_importances.reset_index(inplace=True, drop=True)
 gbrf_feature_importances.to_csv('fi_gbrf_1st_strategy' )
train = pd.read_csv(f'{DATA_DIR}/train.csv' )<set_options>
weight = sum(y.values==0)/sum(y.values==1)
 xgb_gbrf = xgb.XGBClassifier(n_estimators=818, random_state=0, objective='binary:logistic', scale_pos_weight=weight,
learning_rate=0.15, max_depth=2, subsample=0.7, min_child_weight=500, colsample_bytree = 0.2,
reg_lambda = 3.5, reg_alpha=1.5, num_parallel_tree= 5 )
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
def get_img(path):
im_bgr = cv2.imread(path)
 im_rgb = im_bgr[:, :, ::-1]
return im_rgb<categorify>
xgb_gbrf.fit(X, y, eval_set= [(X, y)], eval_metric=['auc'], verbose=False )
class CassavaDataset(Dataset):
def __init__(self, df, data_root, transforms=None, output_label=True):
super().__init__()
self.df = df.reset_index(drop=True ).copy()
self.transforms = transforms
self.data_root = data_root
self.output_label = output_label
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index: int):
if self.output_label:
target = self.df.iloc[index]['label']
img = get_img(f"{self.data_root}/{self.df.loc[index]['image_id']}")
 if self.transforms:
img = self.transforms(image=img)['image']
if self.output_label:
return img, target
else:
return img<choose_model_class>
y_test = xgb_gbrf.predict_proba(X_test)[:,1]
output = pd.DataFrame({'id':sample_submission.index, 'target':y_test})
 output
class CassvaImgClassifier(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):
super().__init__()
self.model = timm.create_model(model_arch, pretrained=pretrained)
 n_features = self.model.classifier.in_features
self.model.classifier = nn.Linear(n_features, n_class)
 def forward(self, x):
x = self.model(x)
 return x<find_best_model_class>
output.to_csv('output_gbrf_1st_strategy.csv', index=False )
def inference_one_epoch(model, data_loader, device):
model.eval()
image_preds_all = []
pbar = tqdm(enumerate(data_loader), total=len(data_loader))
for step,(imgs)in pbar:
imgs = imgs.to(device ).float()
image_preds = model(imgs)
 image_preds_all += [torch.softmax(image_preds, 1 ).detach().cpu().numpy() ]
image_preds_all = np.concatenate(image_preds_all, axis=0)
 return image_preds_all<split>
df_total = pd.concat([df.drop('target', axis=1), test])
 one_hot_cols = ['day','month']
X_cat = df_total[one_hot_cols].astype(str ).copy()
one_hot_enc = OneHotEncoder()
X_cat_1hot = one_hot_enc.fit_transform(X_cat)
 X_cat_1hot = pd.DataFrame(X_cat_1hot.toarray() , index=X_cat.index,
columns = pd.Series(one_hot_enc.get_feature_names() ).str.replace('x0', 'day' ).str.replace('x1', 'month'))
X_cat_1hot.drop(['day_' + X_cat['day'].mode() [0], 'month_' + X_cat['month'].mode() [0]], axis=1, inplace=True)
 X_lr_total = pd.concat([X, X_test])
 X_lr_total = pd.concat([X_lr_total.drop(['sin_day', 'cos_day', 'sin_month', 'cos_month'], axis=1), X_cat_1hot], axis=1)
 X_lr_total['bin_2_nan'] = df_total.bin_2.isna().astype(int)
 X_lr_total['bin_4_nan'] = df_total.bin_4.isna().astype(int)
 X_lr = X_lr_total.iloc[:600000]
X_test_lr = X_lr_total.iloc[600000:1000000]
%%time
if __name__ == '__main__':
seed_everything(CFG['seed'])
 stratifiedKFold = StratifiedKFold(n_splits=CFG['fold_num'])
 folds = stratifiedKFold.split(np.arange(train.shape[0]), train.label.values)
 for fold,(trn_idx, val_idx)in enumerate(folds):
if fold > 0:
break
print(f'Inference fold {fold} started')
 valid_ = train.loc[val_idx,:].reset_index(drop=True)
 valid_ds = CassavaDataset(
 valid_, f'{DATA_DIR}/train_images/',
transforms=get_inference_transforms() , output_label=False)
 test = pd.DataFrame()
test['image_id'] = list(os.listdir(f'{DATA_DIR}/test_images/'))
test_ds = CassavaDataset(
 test, f'{DATA_DIR}/test_images/',
transforms=get_inference_transforms() , output_label=False)
 val_loader = torch.utils.data.DataLoader(
 valid_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 tst_loader = torch.utils.data.DataLoader(
 test_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 device = torch.device(CFG['device'])
 model = CassvaImgClassifier(CFG['model_arch'], train.label.nunique() ).to(device)
 val_preds = []
tst_preds = []
for i, epoch in enumerate(CFG['used_epochs']):
model.load_state_dict(
 torch.load(f'{MODEL_DIR}/{CFG["model_arch"]}_fold_{fold}_{epoch}'))
with torch.no_grad() :
sum_weights = sum(CFG['weights'])
 for _ in range(CFG['tta']):
val_image_preds = inference_one_epoch(model, val_loader, device)
 val_preds += [CFG['weights'][i] / sum_weights / CFG['tta']*val_image_preds]
tst_image_preds = inference_one_epoch(model, tst_loader, device)
 tst_preds += [CFG['weights'][i] / sum_weights / CFG['tta']*tst_image_preds]
val_preds = np.mean(val_preds, axis=0)
 tst_preds = np.mean(tst_preds, axis=0)
 print(f'fold {fold} validation loss = {log_loss(valid_.label.values, val_preds):.5f}')
 print(f'fold {fold} validation accuracy = {(valid_.label.values==np.argmax(val_preds, axis=1)).mean() :.5f}')
 del model
torch.cuda.empty_cache()<save_to_csv>
start = datetime.now()
X_train, X_valid, y_train, y_valid = train_test_split(X_lr, y, test_size=0.2, stratify=y, random_state=42)
 weight = sum(y_train.values==0)/sum(y_train.values==1)
 gbrf = xgb.XGBClassifier(n_estimators=3000, random_state=0, objective='binary:logistic', scale_pos_weight=weight,
learning_rate=0.10, max_depth=4, subsample=0.7, min_child_weight=500,
colsample_bytree = 0.2, reg_lambda = 3.5, reg_alpha=1.5, num_parallel_tree= 5)
 gbrf.fit(X_train, y_train, eval_set= [(X_train, y_train),(X_valid, y_valid)], eval_metric=['auc'],
early_stopping_rounds=100, verbose=False)
 y_pred = gbrf.predict_proba(X_train)
 print("Train AUROC = ", roc_auc_score(y_train, y_pred[:,1]))
y_pred = gbrf.predict_proba(X_valid)
 print("Validation AUROC = ", roc_auc_score(y_valid, y_pred[:,1]))
runtime = datetime.now() - start
print('
Runtime ', runtime )
test['label'] = np.argmax(tst_preds, axis=1)
 test.to_csv('submission.csv', index=False )<install_modules>
xgbfir.saveXgbFI(gbrf, feature_names=X_lr.columns, MaxInteractionDepth=3, OutputXlsxFile='fi_gbrf_2nd_strategy.xlsx' )
!pip install --quiet /kaggle/input/kerasapplications
!pip install --quiet /kaggle/input/efficientnet-git<import_modules>
xls = pd.ExcelFile('fi_gbrf_2nd_strategy.xlsx' )
import math, re, os
import tensorflow as tf
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from kaggle_datasets import KaggleDatasets
from tensorflow import keras
from functools import partial
from sklearn.model_selection import train_test_split
from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau
from efficientnet.tfkeras import EfficientNetB3, EfficientNetB4, EfficientNetB5
import tensorflow.keras.backend as K<define_variables>
fi_depth_1 = pd.read_excel(xls, 'Interaction Depth 1')
 fi_depth_2 = pd.read_excel(xls, 'Interaction Depth 2')
 fi_depth_3 = pd.read_excel(xls, 'Interaction Depth 3' )
AUTOTUNE = tf.data.experimental.AUTOTUNE
WORK_PATH = '.. /input/cassava-leaf-disease-classification'
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
IMAGE_SIZE = [512, 512]
CHANNELS = 3
CLASSES = ['0', '1', '2', '3', '4']
EPOCHS = 30<categorify>
X_lr_total['int_1'] = X_lr_total['ord_0'] * X_lr_total['ord_3'] * X_lr_total['ord_5']
X_lr_total.drop(['bin_3', 'x1_Square', 'x2_Cat', 'x4_Oboe', 'day_5.0'], axis=1, inplace=True)
 X_lr = X_lr_total.iloc[:600000]
X_test_lr = X_lr_total.iloc[600000:1000000]
def decode_image(image):
image = tf.image.decode_jpeg(image, channels=CHANNELS)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image<categorify>
start = datetime.now()
train_scores = []
val_scores = []
y_validations = []
y_predictions = []
for index_train, index_val in StratifiedKFold(n_splits=5, random_state=42, shuffle=True ).split(X_lr, y):
X_train = X_lr.iloc[index_train]
y_train = y.iloc[index_train]
X_valid = X_lr.iloc[index_val]
y_valid = y.iloc[index_val]
lr = LogisticRegression(random_state=0, max_iter=10000, fit_intercept=True, class_weight='balanced',
penalty='l2', C=0.4, verbose=0)
 scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
 X_valid_scaled = scaler.transform(X_valid)
 lr.fit(X_train_scaled, y_train)
 y_pred_train = lr.predict_proba(X_train_scaled)[:,1]
train_score = roc_auc_score(y_train, y_pred_train)
 train_scores.append(train_score)
 y_pred_valid = lr.predict_proba(X_valid_scaled)[:,1]
val_score = roc_auc_score(y_valid, y_pred_valid)
 val_scores.append(val_score)
 y_validations.append(y_valid)
 y_predictions.append(y_pred_valid)
 display_scores(train_scores, val_scores, y_validations, y_predictions)
 runtime = datetime.now() - start
print('
Runtime ', runtime )
def read_tfrecord(example, labeled):
tfrecord_format = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64)
 } if labeled else {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string)
 }
example = tf.io.parse_single_example(example, tfrecord_format)
 image = decode_image(example['image'])
 if labeled:
label = tf.cast(example['target'], tf.int32)
 return image, label
idnum = example['image_name']
return image, idnum<create_dataframe>
lr = LogisticRegression(random_state=0, max_iter=10000, fit_intercept=True, class_weight='balanced', penalty='l2',
C=0.4, verbose=0)
 scaler = StandardScaler()
X_lr_scaled = scaler.fit_transform(X_lr)
 X_test_lr_scaled = scaler.transform(X_test_lr)
 lr.fit(X_lr_scaled, y )
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTOTUNE)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTOTUNE)
 return dataset<define_variables>
y_test = lr.predict_proba(X_test_lr_scaled)[:,1]
output = pd.DataFrame({'id':sample_submission.index, 'target':y_test})
 output
TEST_FILENAMES = tf.io.gfile.glob(WORK_PATH + '/test_tfrecords/ld_test*.tfrec' )<create_dataframe>
output.to_csv('output_lr_2nd_strategy.csv', index=False )
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTOTUNE)
 return dataset<choose_model_class>
start = datetime.now()
chi2_target = pd.DataFrame(columns=['Cochran_criterion', 'Chi2', 'p_value'])
 for col in df.columns.drop('target'):
feature = df[col]
chi2_target.loc[col + '_target'] = chi2_test(feature, df.target)
 chi2_target['rejected'] = chi2_target['Cochran_criterion'] &(chi2_target['p_value']<0.05)
 chi2_target.sort_values('p_value', ascending=True, inplace=True)
 chi2_target.to_csv('chi2_target_tests.csv', index=False)
 print(chi2_target, '
')
 runtime = datetime.now() - start
print('Runtime ', runtime )
with strategy.scope() :
base_model = EfficientNetB5(weights=None, include_top=False, input_shape =(None, None, 3), pooling='avg')
 model = tf.keras.Sequential([
base_model,
tf.keras.layers.Dropout (.3),
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 model.compile(
 optimizer=tf.keras.optimizers.Adam(lr = 0.001),
loss='sparse_categorical_crossentropy',
metrics=['acc'] )<load_pretrained>
for col in ['nom_'+str(i)for i in range(5,10)]:
rare_categories = df[col].value_counts().iloc[np.where(df[col].value_counts() <100)].index
print("The predictor " + col + " has ", rare_categories.size, " rare categories among its ", df[col].unique().size,
" categories." )
model.load_weights('.. /input/cassava-leaf-disease-tpu-efficientnetb4/EffNetB5_best_weights.h5' )<data_type_conversions>
for col in df.columns.drop('target'):
unknowns = set(test[col].astype(str ).unique())- set(df[col].astype(str ).unique())
 print('The predictor ', col, ' contains ', len(unknowns), ' value present only in the test set.' )
def to_float32(image, label):
return tf.cast(image, tf.float32), label<count_values>
start = datetime.now()
train_scores = []
val_scores = []
y_validations = []
y_predictions = []
X_nb = df.drop(['bin_3', 'nom_6', 'target'], axis=1 ).astype(str)
 for index_train, index_val in StratifiedKFold(n_splits=5, random_state=42, shuffle=True ).split(X_nb, y):
X_train = X_nb.iloc[index_train].copy()
y_train = y.iloc[index_train].copy()
X_valid = X_nb.iloc[index_val].copy()
y_valid = y.iloc[index_val].copy()
for col in ['nom_5', 'nom_7', 'nom_8', 'nom_9']:
rare_categories = X_train[col].value_counts().iloc[np.where(X_train[col].value_counts() <100)].index
X_train.loc[:,col] = X_train.loc[:,col].replace(rare_categories, 'nan')
 X_valid.loc[:,col] = X_valid.loc[:,col].replace(rare_categories, 'nan')
 oe = OrdinalEncoder()
X_train_oe = oe.fit_transform(X_train)
 X_val_oe = oe.transform(X_valid)
 X_train_oe += 2
X_val_oe += 2
nb = CategoricalNB(alpha = 3.4)
 nb.fit(X_train_oe, y_train)
 y_pred = nb.predict_proba(X_train_oe)
 train_score = roc_auc_score(y_train, y_pred[:,1])
 train_scores.append(train_score)
 y_pred = nb.predict_proba(X_val_oe)
 val_score = roc_auc_score(y_valid, y_pred[:,1])
 val_scores.append(val_score)
 y_validations.append(y_valid)
 y_predictions.append(y_pred[:,1])
 display_scores(train_scores, val_scores, y_validations, y_predictions)
 runtime = datetime.now() - start
print('
Runtime ', runtime )
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES )<save_to_csv>
oe = OrdinalEncoder()
X_nb = df.drop(['bin_3', 'nom_6', 'target'], axis=1 ).astype(str)
 X_test_nb = test.drop(['bin_3', 'nom_6'], axis=1 ).astype(str)
 for col in ['nom_5', 'nom_7', 'nom_8', 'nom_9']:
rare_categories = X_nb[col].value_counts().iloc[np.where(X_nb[col].value_counts() <100)].index
X_nb[col] = X_nb[col].replace(rare_categories, 'nan')
 X_test_nb[col] = X_test_nb[col].replace(rare_categories, 'nan')
 X_oe = oe.fit_transform(X_nb)
 X_test_oe = oe.transform(X_test_nb)
 X_oe += 2
X_test_oe += 2
test_ds = get_test_dataset(ordered=True ).map(to_float32)
 print('Computing predictions...')
 test_images_ds = test_ds.map(lambda image, idnum: image)
 probabilities = model.predict(test_images_ds)
 test_preds = np.argmax(probabilities, axis=-1)
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
image_names = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 submission = pd.DataFrame({'image_id': image_names, 'label': test_preds})
 submission.to_csv('submission.csv', index=False)
 display(submission.head() )<define_variables>
start = datetime.now()
nb = CategoricalNB(alpha = 3.4)
 nb.fit(X_oe, df.target)
 y_pred = nb.predict_proba(X_oe)
 train_score = roc_auc_score(df.target, y_pred[:,1])
 print('AUROC on the entire training set: ', train_score)
 runtime = datetime.now() - start
print('
Runtime ', runtime )
package_path = '.. /input/pytorch-image-models/pytorch-image-models-master' 
<import_modules>
y_test = nb.predict_proba(X_test_oe)[:,1]
output = pd.DataFrame({'id':sample_submission.index, 'target':y_test})
 output
from glob import glob
from sklearn.model_selection import GroupKFold, StratifiedKFold
import cv2
from skimage import io
import torch
from torch import nn
import os
from datetime import datetime
import time
import random
import cv2
import torchvision
from torchvision import transforms
import pandas as pd
import numpy as np
from tqdm import tqdm
import matplotlib.pyplot as plt
from torch.utils.data import Dataset,DataLoader
from torch.utils.data.sampler import SequentialSampler, RandomSampler
from torch.cuda.amp import autocast, GradScaler
import sklearn
import warnings
import joblib
from sklearn.metrics import roc_auc_score, log_loss
from sklearn import metrics
import warnings
import cv2
import pydicom
import timm
from scipy.ndimage.interpolation import zoom
from sklearn.metrics import log_loss<init_hyperparams>
output.to_csv('output_nb_3rd_strategy.csv', index=False )
CFG = {
'fold_num': 10,
'seed': 719,
'model_arch': 'tf_efficientnet_b3_ns',
'img_size': 512,
'epochs': 32,
'train_bs': 28,
'valid_bs': 32,
'lr': 1e-4,
'num_workers': 4,
'accum_iter': 1,
'verbose_step': 1,
'device': 'cuda:0',
'tta': 1,
'used_epochs': [6,7,8,9],
'weights': [1,1,1,1]
}<load_from_csv>
output_gbrf = pd.read_csv('output_gbrf_1st_strategy.csv')
 output_lr = pd.read_csv('output_lr_2nd_strategy.csv')
 output_nb = pd.read_csv('output_nb_3rd_strategy.csv' )
train = pd.read_csv('.. /input/cassava-leaf-disease-classification/train.csv')
 train.head()<count_values>
mean = output_nb
mean.target =(output_gbrf.target * 0.78560 + output_lr.target * 0.78552 + output_nb.target * 0.78477)/(0.78560 + 0.78552 + 0.78477 )
<load_from_csv><EOS> 
mean.to_csv('ouput_ensemble.csv', index=False )