Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<load_from_disk>
pd.set_option('display.max_rows', 500)
 pd.set_option('display.max_columns', 500)
 pd.set_option('display.width', 1000)
 %matplotlib inline
sns.set(style="whitegrid")
 warnings.filterwarnings("ignore")
 gc.collect()
SEED = 29082013
os.environ['PYTHONHASHSEED']=str(SEED)
 np.random.seed(SEED)
 rn.seed(SEED)
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
with open(os.path.join(WORK_DIR, "label_num_to_disease_map.json")) as file:
print(json.dumps(json.loads(file.read()), indent=4))<load_from_csv>
train = pd.read_csv('/kaggle/input/titanic/train.csv')
 display(train.head(10))
display(train.tail(10))
test = pd.read_csv('/kaggle/input/titanic/test.csv')
 display(test.head(10))
sub= pd.read_csv('/kaggle/input/titanic/gender_submission.csv')
 display(sub.head(3))
train_labels = pd.read_csv(os.path.join(WORK_DIR, "train.csv"))
train_labels.head()<define_variables>
train['Survived'].value_counts(normalize=True, dropna=False )
BATCH_SIZE = 8
STEPS_PER_EPOCH = len(train_labels)*0.8 / BATCH_SIZE
VALIDATION_STEPS = len(train_labels)*0.2 / BATCH_SIZE
EPOCHS = 5
TARGET_SIZE = 512<data_type_conversions>
train.isnull().sum()
train_labels.label = train_labels.label.astype('str')
 train_datagen = ImageDataGenerator(validation_split = 0.2,
preprocessing_function = None,
rotation_range = 45,
zoom_range = 0.2,
horizontal_flip = True,
vertical_flip = True,
fill_mode = 'nearest',
shear_range = 0.1,
height_shift_range = 0.1,
width_shift_range = 0.1)
 train_generator = train_datagen.flow_from_dataframe(train_labels,
directory = os.path.join(WORK_DIR, "train_images"),
subset = "training",
x_col = "image_id",
y_col = "label",
target_size =(TARGET_SIZE, TARGET_SIZE),
batch_size = BATCH_SIZE,
class_mode = "sparse")
 validation_datagen = ImageDataGenerator(validation_split = 0.2)
 validation_generator = validation_datagen.flow_from_dataframe(train_labels,
directory = os.path.join(WORK_DIR, "train_images"),
subset = "validation",
x_col = "image_id",
y_col = "label",
target_size =(TARGET_SIZE, TARGET_SIZE),
batch_size = BATCH_SIZE,
class_mode = "sparse" )<prepare_x_and_y>
train.isnull().sum()
generator = train_datagen.flow_from_dataframe(train_labels.iloc[20:21],
directory = os.path.join(WORK_DIR, "train_images"),
x_col = "image_id",
y_col = "label",
target_size =(TARGET_SIZE, TARGET_SIZE),
batch_size = BATCH_SIZE,
class_mode = "sparse")
 aug_images = [generator[0][0][0]/255 for i in range(10)]
fig, axes = plt.subplots(2, 5, figsize =(20, 10))
axes = axes.flatten()
for img, ax in zip(aug_images, axes):
ax.imshow(img)
 ax.axis('off')
 plt.tight_layout()
plt.show()<choose_model_class>
def null_verificator(data):
if data.isnull().any().any() :
view_info = pd.DataFrame(
 pd.concat(
 [data.isnull().any() ,
data.isnull().sum() ,
data.dtypes],
axis=1)
)
 view_info.columns = ['Nulos', 'Cantidad', 'Tipo Col']
size = data.shape[0]
view_info['Porcentaje'] = view_info['Cantidad'].apply(
 lambda x: str(np.round(0 if not x else x*100 / size, 2)) + ' %' 
)
 return view_info
else:
return pd.DataFrame.from_dict({'msje':"DATA LIMPIA DE NULOS"}, orient='index')
classes_to_predict = sorted(train_labels.label.unique())
 dropout_rate = 0.3
def create_model() :
model = models.Sequential()
model.add(EfficientNetB4(include_top = False, weights = None,
input_shape =(TARGET_SIZE, TARGET_SIZE, 3)))
 model.add(layers.GlobalAveragePooling2D())
 model.add(Dropout(dropout_rate))
model.add(Dense(len(classes_to_predict), activation="softmax"))
model.compile(optimizer = Adam(lr = 0.001),
loss = "sparse_categorical_crossentropy",
metrics = ["acc"])
 return model<choose_model_class>
display(null_verificator(train))
display(null_verificator(test))
print('Our EfficientNet CNN has %d layers' %len(model.layers))<load_pretrained>
train['Pclass'] = train['Pclass'].astype(str)
 test['Pclass'] = test['Pclass'].astype(str )
model.load_weights('.. /input/cassava-leaf-keras-efficientnetb-baseline/best_baseline_model.h5' )<choose_model_class>
for col in ['Pclass', 'Sex', 'Embarked']:
compara = pd.concat(
 [train[col].value_counts(dropna=False, normalize=True ).sort_index() ,
test[col].value_counts(dropna=False, normalize=True ).sort_index() ], axis=1 
)
 compara.columns = ['train-'+col, 'test-'+col]
display(compara)
 del compara
model_save = ModelCheckpoint('./EffNetB4_best_weights.h5',
save_best_only = True,
save_weights_only = True,
monitor = 'val_loss',
mode = 'min', verbose = 1)
 early_stop = EarlyStopping(monitor = 'val_loss', min_delta = 0.001,
patience = 5, mode = 'min', verbose = 1,
restore_best_weights = True)
 reduce_lr = ReduceLROnPlateau(monitor = 'val_loss', factor = 0.3,
patience = 2, min_delta = 0.001,
mode = 'min', verbose = 1)
 history = model.fit(
 train_generator,
steps_per_epoch = STEPS_PER_EPOCH,
epochs = EPOCHS,
validation_data = validation_generator,
validation_steps = VALIDATION_STEPS,
callbacks = [model_save, early_stop, reduce_lr]
)<train_model>
cols_cat = ['Pclass', 'Sex', 'Embarked']
col_target = 'Survived'
avg_acc = sum(history.history['acc'])/len(history.history['acc'])
 avg_val_acc = sum(history.history['val_acc'])/len(history.history['val_acc'])
 avg_loss = sum(history.history['loss'])/len(history.history['loss'])
 avg_val_loss = sum(history.history['val_loss'])/len(history.history['val_loss'])
 print('Training produced an average accuracy of %.3f and average loss of %.3f' %(avg_acc,avg_loss))
print('Validation produced an average accuracy of %.3f and average loss of %.3f' %(avg_val_acc,avg_val_loss))<load_from_csv>
mean_age_train = train.groupby(by=['Pclass', 'Sex'])['Age'].median().reset_index()
mean_age_train.columns = ['Pclass', 'Sex', 'mean_age']
mean_age_train
ss = pd.read_csv(os.path.join(WORK_DIR, "sample_submission.csv"))
ss<predict_on_test>
train = train.merge(mean_age_train, how='left', on=['Pclass', 'Sex'])
 test = test.merge(mean_age_train, how='left', on=['Pclass', 'Sex'])
 train['Age'] = train['Age'].combine_first(train['mean_age'])
 test['Age'] = test['Age'].combine_first(train['mean_age'])
 del train['mean_age']
del test['mean_age']
train.shape, test.shape
preds = []
for image_id in ss.image_id:
image = Image.open(os.path.join(WORK_DIR, "test_images", image_id))
image = image.resize(( TARGET_SIZE, TARGET_SIZE))
image = np.expand_dims(image, axis = 0)
 preds.append(np.argmax(model.predict(image)))
 ss['label'] = preds
ss<save_to_csv>
mean_fare_train = train.groupby(by=['Pclass', 'Sex'])['Fare'].median().reset_index()
mean_fare_train.columns = ['Pclass', 'Sex', 'mean_fare']
test = test.merge(mean_fare_train, how='left', on=['Pclass', 'Sex'])
 test['Fare'] = test['Fare'].combine_first(test['mean_fare'])
 del test['mean_fare']
train.shape, test.shape
ss.to_csv('submission.csv', index = False )<install_modules>
train['Cabin'].fillna('X', inplace=True)
 test['Cabin'].fillna('X', inplace=True)
 train['Cabin'].isnull().sum() , test['Cabin'].isnull().sum()
!pip install --quiet /kaggle/input/kerasapplications
!pip install --quiet /kaggle/input/efficientnet-git<set_options>
train['Embarked'].fillna('S', inplace=True)
 display(null_verificator(train))
display(null_verificator(test))
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
seed = 0
seed_everything(seed)
 warnings.filterwarnings('ignore' )<set_options>
train['Mr'] = train['Name'].apply(lambda _: int('Mr' in _))
test['Mr'] = test['Name'].apply(lambda _: int('Mr' in _))
train['Mrs'] = train['Name'].apply(lambda _: int('Mrs' in _))
test['Mrs'] = test['Name'].apply(lambda _: int('Mrs' in _))
train['Miss'] = train['Name'].apply(lambda _: int('Miss' in _))
test['Miss'] = test['Name'].apply(lambda _: int('Miss' in _))
train['Dr'] = train['Name'].apply(lambda _: int('Dr' in _))
test['Dr'] = test['Name'].apply(lambda _: int('Dr' in _))
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print(f'Running on TPU {tpu.master() }')
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )<define_variables>
train['words_in_name'] = train['Name'].apply(lambda _: len(_.split()))
test['words_in_name'] = test['Name'].apply(lambda _: len(_.split()))
view_numeric(train, 'words_in_name', col_target )
BATCH_SIZE = 8 * REPLICAS
HEIGHT = 512
WIDTH = 512
CHANNELS = 3
N_CLASSES = 5
TTA_STEPS = 8<define_search_space>
del train['Name']
del test['Name']
def data_augment(image, label):
p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_pixel_1 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_pixel_2 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_pixel_3 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 if p_spatial >.75:
image = tf.image.transpose(image)
 if p_rotate >.75:
image = tf.image.rot90(image, k=3)
 elif p_rotate >.5:
image = tf.image.rot90(image, k=2)
 elif p_rotate >.25:
image = tf.image.rot90(image, k=1)
 if p_pixel_1 >=.4:
image = tf.image.random_saturation(image, lower=.7, upper=1.3)
 if p_pixel_2 >=.4:
image = tf.image.random_contrast(image, lower=.8, upper=1.2)
 if p_pixel_3 >=.4:
image = tf.image.random_brightness(image, max_delta=.1)
 if p_crop >.7:
if p_crop >.9:
image = tf.image.central_crop(image, central_fraction=.7)
 elif p_crop >.8:
image = tf.image.central_crop(image, central_fraction=.8)
 else:
image = tf.image.central_crop(image, central_fraction=.9)
 elif p_crop >.4:
crop_size = tf.random.uniform([], int(HEIGHT*.8), HEIGHT, dtype=tf.int32)
 image = tf.image.random_crop(image, size=[crop_size, crop_size, CHANNELS])
 return image, label<normalization>
display(train['Cabin'].value_counts())
 display(train['Ticket'].value_counts() )
def get_name(file_path):
parts = tf.strings.split(file_path, os.path.sep)
 name = parts[-1]
return name
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
return image
def center_crop(image):
image = tf.reshape(image, [600, 800, CHANNELS])
 h, w = image.shape[0], image.shape[1]
if h > w:
image = tf.image.crop_to_bounding_box(image,(h - w)// 2, 0, w, w)
 else:
image = tf.image.crop_to_bounding_box(image, 0,(w - h)// 2, h, h)
 image = tf.image.resize(image, [HEIGHT, WIDTH])
 return image
def resize_image(image, label):
image = tf.image.resize(image, [HEIGHT, WIDTH])
 image = tf.reshape(image, [HEIGHT, WIDTH, CHANNELS])
 return image, label
def process_path(file_path):
name = get_name(file_path)
 img = tf.io.read_file(file_path)
 img = decode_image(img)
 return img, name
def get_dataset(files_path, shuffled=False, tta=False, extension='jpg'):
dataset = tf.data.Dataset.list_files(f'{files_path}*{extension}', shuffle=shuffled)
 dataset = dataset.map(process_path, num_parallel_calls=AUTO)
 if tta:
dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.map(resize_image, num_parallel_calls=AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n )<train_model>
train.groupby(by=['Cabin'])['PassengerId'].size()
model_path_list = glob.glob('/kaggle/input/cassava-leaf-disease-training-with-tpu-v2-pods/*.h5')
 model_path_list.sort()
print('Models to predict:')
 print(*model_path_list, sep='
' )<choose_model_class>
def decision(cabin_val, num):
if cabin_val == 'X':
return -1
elif num > 1:
return 1
else:
return 0
def detect_share(data, col_analysis, new_column):
_g = data.groupby(by=[col_analysis] ).agg({
'PassengerId': 'size',
} ).reset_index()
_g[new_column] = _g[[col_analysis, 'PassengerId']].apply(
 lambda _: decision(_[0], _[1]), axis=1 
)
 data = data.merge(_g.drop(['PassengerId'], axis=1), on=col_analysis, how='left')
 del _g
return data
def model_fn(input_shape, N_CLASSES):
inputs = L.Input(shape=input_shape, name='input_image')
 base_model = efn.EfficientNetB4(input_tensor=inputs,
include_top=False,
weights=None,
pooling='avg')
 x = L.Dropout (.5 )(base_model.output)
 output = L.Dense(N_CLASSES, activation='softmax', name='output' )(x)
 model = Model(inputs=inputs, outputs=output)
 return model
with strategy.scope() :
model = model_fn(( None, None, CHANNELS), N_CLASSES)
 model.summary()<predict_on_test>
del train['Ticket']
del test['Ticket']
del train['Cabin']
del test['Cabin']
files_path = f'{database_base_path}test_images/'
test_size = len(os.listdir(files_path))
test_preds = np.zeros(( test_size, N_CLASSES))
for model_path in model_path_list:
print(model_path)
 K.clear_session()
model.load_weights(model_path)
 if TTA_STEPS > 0:
test_ds = get_dataset(files_path, tta=True ).repeat()
ct_steps = TTA_STEPS *(( test_size/BATCH_SIZE)+ 1)
 preds = model.predict(test_ds, steps=ct_steps, verbose=1)[:(test_size * TTA_STEPS)]
preds = np.mean(preds.reshape(test_size, TTA_STEPS, N_CLASSES, order='F'), axis=1)
 test_preds += preds / len(model_path_list)
 else:
test_ds = get_dataset(files_path, tta=False)
 x_test = test_ds.map(lambda image, image_name: image)
 test_preds += model.predict(x_test)/ len(model_path_list)
 test_preds = np.argmax(test_preds, axis=-1)
 test_names_ds = get_dataset(files_path)
 image_names = [img_name.numpy().decode('utf-8')for img, img_name in iter(test_names_ds.unbatch())]<save_to_csv>
train['total_family'] = train['SibSp'] + train['Parch']
test['total_family'] = test['SibSp'] + test['Parch']
view_numeric(train, 'total_family', col_target )
submission = pd.DataFrame({'image_id': image_names, 'label': test_preds})
 submission.to_csv('submission.csv', index=False)
 display(submission.head() )<define_variables>
train['family_greater_than_3'] = train[['SibSp', 'Parch']].apply(lambda _: 1 if _[0] + _[1] > 3 else 0, axis=1)
 test['family_greater_than_3'] = test[['SibSp', 'Parch']].apply(lambda _: 1 if _[0] + _[1] > 3 else 0, axis=1)
 view_cat(train, 'family_greater_than_3', col_target )
Cassava_dir = ".. /input/cassava-leaf-disease-classification/"<load_pretrained>
def _range_edad(_):
if _ < 0:
return 'sin edad'
elif _ < 10:
return 'ninno'
elif _ < 20:
return 'adolescente'
elif _ < 30:
return 'joven'
elif _ < 40:
return 'adulto'
elif _ < 50:
return 'adultomayor'
else:
return 'anciano'
train['range_edad'] = train['Age'].apply(_range_edad)
 test['range_edad'] = test['Age'].apply(_range_edad)
 view_cat(train, 'range_edad', 'Survived' )
model = keras.models.load_model('.. /input/cassava-baseline-weights/best_weights.h5' )<create_dataframe>
y_train = train['Survived'].copy()
train = train.drop(['PassengerId', 'Survived'], axis=1)
 test = test.drop(['PassengerId'], axis=1 )
sub = pd.DataFrame(columns=['image_id','label'] )<load_pretrained>
train = pd.get_dummies(train, drop_first=True, columns=['Sex'])
 test = pd.get_dummies(test, drop_first=True, columns=['Sex'] )
def predict_on_batch(test_list, wpath=Cassava_dir, target_size=(380,380)) :
input_batch=[]
for IMAGE_ID in test_list:
image = tf.keras.preprocessing.image.load_img(os.path.join(wpath, "test_images",IMAGE_ID),
grayscale=False,
color_mode="rgb",
target_size=target_size,
interpolation="nearest")
 input_arr = keras.preprocessing.image.img_to_array(image)
 input_batch.append(input_arr)
 return np.array(input_batch )<define_variables>
train = pd.get_dummies(train, drop_first=False)
 test = pd.get_dummies(test, drop_first=False )
TEST_DIR = '.. /input/cassava-leaf-disease-classification/test_images/'
test_images = os.listdir(TEST_DIR)
 N = len(test_images)
 if N == 1:
BATCH_SIZE = 1
else:
BATCH_SIZE = 16<predict_on_test>
abs(_corr['Survived'] ).sort_values(ascending=False )
for batch_index in range(math.ceil(N/BATCH_SIZE)) :
if batch_index*BATCH_SIZE+BATCH_SIZE < N:
test_X = predict_on_batch(test_images[batch_index*BATCH_SIZE:batch_index*BATCH_SIZE+BATCH_SIZE])
 predictions = model.predict(test_X ).argmax(axis = 1)
 sub_batch = pd.DataFrame({'image_id':test_images[batch_index*BATCH_SIZE:batch_index*BATCH_SIZE+BATCH_SIZE],
'label':list(predictions)})
 else:
test_X = predict_on_batch(test_images[batch_index*BATCH_SIZE:])
 predictions = model.predict(test_X ).argmax(axis = 1)
 sub_batch = pd.DataFrame({'image_id':test_images[batch_index*BATCH_SIZE:],
'label':list(predictions)})
 sub = sub.append(sub_batch, ignore_index=True )<data_type_conversions>
list_var_relevants = ['Age', 'SibSp', 'Fare', 'Mr', 'Mrs', 'shared_cabin', 'family_greater_than_3',
'total_family', 'words_in_name', 'Sex_male', 'Pclass_1', 'Pclass_3', 'Embarked_S',
'range_edad_joven', 'range_edad_ninno']
sub['label'] = sub['label'].astype('int64')
 sub<save_to_csv>
c_values = list(np.logspace(-1, 0.5, 50))
print(c_values )
sub.to_csv('submission.csv', index = False )<define_variables>
param_grid_log = {
'penalty': ['l2'],
'C': c_values,
'class_weight': ['balanced', None],
'max_iter': [75],
'solver': ['lbfgs']
}
kfold_on_rf = StratifiedKFold(
 n_splits=3,
shuffle=False,
random_state=SEED 
)
 model_log = LogisticRegression(random_state=SEED, n_jobs = 4 )
package_path = '.. /input/pytorch-image-models/pytorch-image-models-master' 
<import_modules>
def apply_grid(X_train, y_train, model, param_grid, kfold_):
grid = RandomizedSearchCV(
 model,
{k: [v] if not isinstance(v, list)else v for k, v in param_grid.items() },
cv=kfold_,
n_jobs=-1,
scoring='accuracy',
verbose=1,
n_iter=1500 
)
 grid.fit(X_train, y_train)
 print(grid.best_score_, end=' / ')
 return grid.best_estimator_, grid.best_score_, grid.best_params_
from glob import glob
from sklearn.model_selection import GroupKFold, StratifiedKFold
import cv2
from skimage import io
import torch
from torch import nn
import os
from datetime import datetime
import time
import random
import cv2
import torchvision
from torchvision import transforms
import pandas as pd
import numpy as np
from tqdm import tqdm
import matplotlib.pyplot as plt
from torch.utils.data import Dataset,DataLoader
from torch.utils.data.sampler import SequentialSampler, RandomSampler
from torch.cuda.amp import autocast, GradScaler
import sklearn
import warnings
import joblib
from sklearn.metrics import roc_auc_score, log_loss
from sklearn import metrics
import warnings
import cv2
import pydicom
import timm
from scipy.ndimage.interpolation import zoom
from sklearn.metrics import log_loss<init_hyperparams>
result_0_1_train = best_model.predict(train[list_var_relevants])
 acu = accuracy_score(y_train, result_0_1_train)
 rec = recall_score(y_train, result_0_1_train)
 print(acu, rec)
 print(classification_report(y_train, result_0_1_train))
CFG = {
'fold_num': 5,
'seed': 719,
'model_arch': 'tf_efficientnet_b4_ns',
'img_size': 512,
'epochs': 10,
'train_bs': 32,
'valid_bs': 32,
'lr': 1e-4,
'num_workers': 4,
'accum_iter': 1,
'verbose_step': 1,
'device': 'cuda:0',
'tta': 3,
'used_epochs': [6,7,8,9],
'weights': [1,1,1,1]
}<load_from_csv>
sub['Survived'] = best_model.predict(test[list_var_relevants])
 sub['Survived'].value_counts()
train = pd.read_csv('.. /input/cassava-leaf-disease-classification/train.csv')
 train.head()<count_values>
_date = str(datetime.now() ).split('.')[0].replace('-', '_' ).replace(' ', '_' ).replace(':', '_')
 sub.to_csv('result_log_lassocv_{}_{}_{}.csv'.format(
 round(acu, 4),round(rec, 4),_date 
), index=False )
train.label.value_counts()<load_from_csv>
result_prob_train = best_model.predict_proba(train[list_var_relevants])[:,1]
accuracy_score(y_train, np.array([0 if _ < 0.5 else 1 for _ in result_prob_train]))
submission = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 submission.head()<categorify>
optimization = differential_evolution(
 lambda c: -1*accuracy_score(y_train, np.array([0 if _ < c[0] else 1 for _ in result_prob_train])) ,
[(0, 1)] 
)
 optimization
class CassavaDataset(Dataset):
def __init__(
 self, df, data_root, transforms=None, output_label=True 
):
super().__init__()
self.df = df.reset_index(drop=True ).copy()
self.transforms = transforms
self.data_root = data_root
self.output_label = output_label
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index: int):
if self.output_label:
target = self.df.iloc[index]['label']
path = "{}/{}".format(self.data_root, self.df.iloc[index]['image_id'])
 img = get_img(path)
 if self.transforms:
img = self.transforms(image=img)['image']
if self.output_label == True:
return img, target
else:
return img<import_modules>
result_0_1_train_opt = np.array([0 if _ < optimization["x"][0] else 1 for _ in result_prob_train])
 acu = accuracy_score(y_train, result_0_1_train_opt)
 rec = recall_score(y_train, result_0_1_train_opt)
 print(acu, rec)
 print(classification_report(y_train, result_0_1_train_opt))
HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize 
)
 HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize 
)
 def get_train_transforms() :
return Compose([
RandomResizedCrop(CFG['img_size'], CFG['img_size']),
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
ShiftScaleRotate(p=0.5),
HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),
RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
CoarseDropout(p=0.5),
Cutout(p=0.5),
ToTensorV2(p=1.0),
], p=1.)
def get_valid_transforms() :
return Compose([
CenterCrop(CFG['img_size'], CFG['img_size'], p=1.) ,
Resize(CFG['img_size'], CFG['img_size']),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.)
def get_inference_transforms() :
return Compose([
RandomResizedCrop(CFG['img_size'], CFG['img_size']),
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),
RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.)<choose_model_class>
result_0_1_test_opt = np.array(
 [0 if _ < optimization["x"][0] else 1 for _ in best_model.predict_proba(test[list_var_relevants])[:,1]] 
)
 sub['Survived'] = result_0_1_test_opt
sub['Survived'].value_counts()
<split><EOS> 
_date = str(datetime.now() ).split('.')[0].replace('-', '_' ).replace(' ', '_' ).replace(':', '_')
 sub.to_csv('result_loglassocv_opt_{}_{}_{}.csv'.format(
 round(acu, 5),round(rec, 5),_date 
), index=False )