Kaggle competition: SIIM-ISIC Melanoma Classification
y_pred = model.predict(test, verbose = 1)
<define_variables>
colab=0
show_files=0
tstamp=0
predictions=[]
for i in range(len(test)) :
a=np.where(y_pred[i] == max(y_pred[i]))
predictions.append(a[0][0] )<save_to_csv>
if colab:
drive.mount('/content/gdrive' )
counter = range(1, len(predictions)+ 1)
 solution = pd.DataFrame({"ImageId": counter, "label": list(predictions)})
 solution.to_csv("digit_recognizer8.csv", index = False )<import_modules>
!pip install -q efficientnet
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from keras.models import Model, load_model
from keras.layers import Conv2D, Input, MaxPooling2D, Dense, Dropout, Flatten
from keras.layers import LeakyReLU
from keras.layers.normalization import BatchNormalization
from keras.preprocessing.image import ImageDataGenerator
from keras.utils.np_utils import to_categorical
from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau
from sklearn.model_selection import train_test_split<load_from_csv>
print("Tensorflow version " + tf.__version__)
 AUTO = tf.data.experimental.AUTOTUNE
if not colab:
train = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 test = pd.read_csv("/kaggle/input/digit-recognizer/test.csv")
 submission = pd.read_csv("/kaggle/input/digit-recognizer/sample_submission.csv")
 X = train.drop(['label'],1 ).values
Y = train['label'].values
x_test = test.values
X = X/255.
 x_test = x_test/255.
 X = X.reshape(-1,28,28,1)
 x_test = x_test.reshape(-1,28,28,1)
 Y = to_categorical(Y)
<split>
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed )
x_train, x_valid, y_train, y_valid = train_test_split(X,Y, test_size=0.1 )<define_search_model>
NAME='EffNB3_512'
NFOLDS=5
NBEST=2
SEED=311
if colab:
PATH=Path('/content/gdrive/My Drive/kaggle/input/siim-isic-melanoma-classification/')
 train=pd.read_csv(PATH/'train.csv.zip')
 else:
PATH=Path('/kaggle/input/siim-isic-melanoma-classification/')
 train=pd.read_csv(PATH/'train.csv')
 test=pd.read_csv(PATH/'test.csv')
 sub=pd.read_csv(PATH/'sample_submission.csv')
 seed_everything(SEED )
def get_model() :
In = Input(shape=(28,28,1))
x = Conv2D(32,(3,3), padding="same" )(In)
 x = LeakyReLU(alpha=0.01 )(x)
 x = Conv2D(32,(3,3), padding="same" )(x)
 x = LeakyReLU(alpha=0.01 )(x)
 x = BatchNormalization()(x)
 x = MaxPooling2D(( 2,2))(x)
 x = Conv2D(64,(3,3), padding="same" )(x)
 x = LeakyReLU(alpha=0.01 )(x)
 x = Conv2D(64,(3,3), padding="same" )(x)
 x = LeakyReLU(alpha=0.01 )(x)
 x = BatchNormalization()(x)
 x = MaxPooling2D(( 2,2))(x)
 x = Dropout(0.2 )(x)
 x = Conv2D(128,(3,3), padding="same" )(x)
 x = LeakyReLU(alpha=0.01 )(x)
 x = Conv2D(128,(3,3), padding="same" )(x)
 x = LeakyReLU(alpha=0.01 )(x)
 x = BatchNormalization()(x)
 x = MaxPooling2D(( 2,2))(x)
 x = Dropout(0.2 )(x)
 x = Flatten()(x)
 x = Dense(512 )(x)
 x = LeakyReLU(alpha=0.01 )(x)
 x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 x = Dense(256 )(x)
 x = LeakyReLU(alpha=0.01 )(x)
 x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 x = Dense(128 )(x)
 x = LeakyReLU(alpha=0.01 )(x)
 x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 Out = Dense(10, activation="softmax" )(x)
 model = Model(In, Out)
 model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"])
 return model
model = get_model()
model.summary()<choose_model_class>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
best_checkpoint = ModelCheckpoint('best.hdf5',monitor = 'val_loss', mode = "min", verbose = 1, save_best_only = True)
 lr_reduction = ReduceLROnPlateau(monitor = 'val_loss', patience = 3, verbose = 1, factor = 0.5, min_lr = 1e-6 )<define_variables>
GCS_PATH = KaggleDatasets().get_gcs_path('siim-isic-melanoma-classification')
 print(GCS_PATH)
 !gsutil ls $GCS_PATH
data_generator = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 epochs = 100
batch_size = 128
train_generator = data_generator.flow(x_train, y_train, batch_size=batch_size)
 valid_generator = data_generator.flow(x_valid, y_valid, batch_size=batch_size )<train_model>
GCS_PATH={}
if colab:
GCS_PATH['train']='gs://kds-8d3ddd90d7523c7ce02205a8ca8dd2d6c1fde5e071c72f852587566c'
GCS_PATH['test']='gs://kds-6c0f251418566e5596a0ab88ef637925c8f18c05edd525d1d741a07f'
else:
GCS_PATH['train']=KaggleDatasets().get_gcs_path('siim-512x512-tfrec-q95')
 GCS_PATH['test']=KaggleDatasets().get_gcs_path('siim-512x512-tfrec-q95-test')
 print(GCS_PATH['train'])
 print(GCS_PATH['test'] )
hist = model.fit_generator(train_generator, epochs=epochs, steps_per_epoch = x_train.shape[0]//batch_size,
validation_data = valid_generator, validation_steps = x_valid.shape[0]//batch_size, callbacks=[best_checkpoint, lr_reduction], verbose=1 )<predict_on_test>
%%time
IMAGE_SIZE = [224, 224]
EPOCHS=17
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
CLASSES = ['benign', 'malignant']
best = load_model("best.hdf5")
 preds = best.predict(x_test, verbose=1)
 preds = np.array([np.argmax(i)for i in preds])
 preds<save_to_csv>
ALL_TRAIN=tf.io.gfile.glob(GCS_PATH['train'] + '/*.tfrec')
 VAL_FNAMES={}
for fn in range(1, NFOLDS+1):
VAL_FNAMES[f"fold_{fn}"]=[path for path in ALL_TRAIN if f"fold_{fn}" in path]
print("Fold", f'{fn}:', len(VAL_FNAMES[f'fold_{fn}']), "elements in total.")
 TRAIN_FNAMES={f'fold_{i}': list(set(ALL_TRAIN)-set(VAL_FNAMES[f'fold_{i}']))
for i in range(1, NFOLDS+1)}
TEST_FNAMES = tf.io.gfile.glob(GCS_PATH['test'] + '/*.tfrec' )
submission['Label'] = preds
submission.to_csv("submission.csv", index=False)
 submission.head()<load_from_csv>
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n )
pd.set_option('mode.chained_assignment', None)
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-3/test.csv")
 train = pd.read_csv(".. /input/covid19-global-forecasting-week-3/train.csv")
 train['Province_State'].fillna('', inplace=True)
 test['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'])
 train = train.sort_values(['Country_Region','Province_State','Date'])
 test = test.sort_values(['Country_Region','Province_State','Date'])
 train[['ConfirmedCases', 'Fatalities']] = train.groupby(['Country_Region', 'Province_State'])[['ConfirmedCases', 'Fatalities']].transform('cummax' )<feature_engineering>
%%time
N_TRAIN_IMGS = {f'fold_{i}': count_data_items(TRAIN_FNAMES[f'fold_{i}'])
 for i in range(1, NFOLDS+1)}
N_VAL_IMGS = {f'fold_{i}': count_data_items(VAL_FNAMES[f'fold_{i}'])
 for i in range(1, NFOLDS+1)}
N_TEST_IMGS = count_data_items(TEST_FNAMES)
 STEPS_PER_EPOCH = {f'fold_{i}': N_TRAIN_IMGS[f'fold_{i}'] // BATCH_SIZE
for i in range(1, NFOLDS+1)}
print("="*75)
 print(f"The number of unlabeled test image is {N_TEST_IMGS}.It is common for all folds.")
 for i in range(1, NFOLDS+1):
print("="*75)
 print(f"Fold {i}: {N_TRAIN_IMGS[f'fold_{i}']} training and {N_VAL_IMGS[f'fold_{i}']} validation images.")
 print("="*75 )
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)))
 feature_day = [1,20,50,100,200,500,1000,5000,10000,15000,20000,50000,100000,200000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature] 
<install_modules>
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
!pip install pmdarima<save_to_csv>
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
"patient_id": tf.io.FixedLenFeature([], tf.string),
"benign_malignant": tf.io.FixedLenFeature([], tf.string),
"age": tf.io.FixedLenFeature([], tf.int64),
"sex_female": tf.io.FixedLenFeature([], tf.int64),
"sex_male": tf.io.FixedLenFeature([], tf.int64),
"sex_unknown": tf.io.FixedLenFeature([], tf.int64),
"site_head/neck": tf.io.FixedLenFeature([], tf.int64),
"site_lower extremity": tf.io.FixedLenFeature([], tf.int64),
"site_oral/genital": tf.io.FixedLenFeature([], tf.int64),
"site_palms/soles": tf.io.FixedLenFeature([], tf.int64),
"site_torso": tf.io.FixedLenFeature([], tf.int64),
"site_unknown": tf.io.FixedLenFeature([], tf.int64),
"site_upper extremity": tf.io.FixedLenFeature([], tf.int64),
"height": tf.io.FixedLenFeature([], tf.int64),
"width": tf.io.FixedLenFeature([], tf.int64),
"target": tf.io.FixedLenFeature([], tf.int64),
"age_scaled": tf.io.FixedLenFeature([], tf.float32),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 data={}
data['image_name']=image_name=tf.cast(example['image_name'], tf.string)
 data['patient_id']=tf.cast(example['patient_id'], tf.string)
 data['age']=tf.cast(example['age'], tf.int32)
 data['sex_female']=tf.cast(example['sex_female'], tf.int32)
 data['sex_male']=tf.cast(example['sex_male'], tf.int32)
 data['sex_unknown']=tf.cast(example['sex_unknown'], tf.int32)
 data['site_head/neck']=tf.cast(example['site_head/neck'], tf.int32)
 data['site_lower extremity']=tf.cast(example['site_lower extremity'], tf.int32)
 data['site_oral/genital']=tf.cast(example['site_oral/genital'], tf.int32)
 data['site_palms/soles']=tf.cast(example['site_palms/soles'], tf.int32)
 data['site_torso']=tf.cast(example['site_torso'], tf.int32)
 data['site_unknown']=tf.cast(example['site_unknown'], tf.int32)
 data['site_upper extremity']=tf.cast(example['site_upper extremity'], tf.int32)
 data['age_scaled']=tf.cast(example['age_scaled'], tf.float32)
 label=tf.cast(example['target'], tf.int32)
 label_name=tf.cast(example['benign_malignant'], tf.string)
 return image, label, data, label_name
df_val = df_val_2
submission = df_val[['ForecastId','ConfirmedCases_hat','Fatalities_hat']]
submission.columns = ['ForecastId','ConfirmedCases','Fatalities']
submission = submission.round({'ConfirmedCases': 0, 'Fatalities': 0})
 submission.to_csv('submission.csv', index=False)
 submission<install_modules>
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
"patient_id": tf.io.FixedLenFeature([], tf.string),
"age": tf.io.FixedLenFeature([], tf.int64),
"sex_female": tf.io.FixedLenFeature([], tf.int64),
"sex_male": tf.io.FixedLenFeature([], tf.int64),
"sex_unknown": tf.io.FixedLenFeature([], tf.int64),
"site_head/neck": tf.io.FixedLenFeature([], tf.int64),
"site_lower extremity": tf.io.FixedLenFeature([], tf.int64),
"site_oral/genital": tf.io.FixedLenFeature([], tf.int64),
"site_palms/soles": tf.io.FixedLenFeature([], tf.int64),
"site_torso": tf.io.FixedLenFeature([], tf.int64),
"site_unknown": tf.io.FixedLenFeature([], tf.int64),
"site_upper extremity": tf.io.FixedLenFeature([], tf.int64),
"height": tf.io.FixedLenFeature([], tf.int64),
"width": tf.io.FixedLenFeature([], tf.int64),
"age_scaled": tf.io.FixedLenFeature([], tf.float32),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 data={}
data['image_name']=image_name=tf.cast(example['image_name'], tf.string)
 data['patient_id']=tf.cast(example['patient_id'], tf.string)
 data['age']=tf.cast(example['age'], tf.int32)
 data['sex_female']=tf.cast(example['sex_female'], tf.int32)
 data['sex_male']=tf.cast(example['sex_male'], tf.int32)
 data['sex_unknown']=tf.cast(example['sex_unknown'], tf.int32)
 data['site_head/neck']=tf.cast(example['site_head/neck'], tf.int32)
 data['site_lower extremity']=tf.cast(example['site_lower extremity'], tf.int32)
 data['site_oral/genital']=tf.cast(example['site_oral/genital'], tf.int32)
 data['site_palms/soles']=tf.cast(example['site_palms/soles'], tf.int32)
 data['site_torso']=tf.cast(example['site_torso'], tf.int32)
 data['site_unknown']=tf.cast(example['site_unknown'], tf.int32)
 data['site_upper extremity']=tf.cast(example['site_upper extremity'], tf.int32)
 data['age_scaled']=tf.cast(example['age_scaled'], tf.float32)
 return image, data
!pip install keras-bert
!pip install keras-rectified-adam<import_modules>
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled
else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
import codecs
import tensorflow as tf
from tqdm import tqdm
from chardet import detect
import keras
from keras_radam import RAdam
from keras import backend as K
from keras_bert import load_trained_model_from_checkpoint
import codecs
from keras_bert import Tokenizer<load_pretrained>
%%time
training_dataset = load_dataset(TRAIN_FNAMES['fold_2'])
 print("Example of the training data:")
 for image, label, data, label_name in training_dataset.take(1):
print("The image batch size:", image.numpy().shape)
 print("Label:", label.numpy())
 print("Label name:", label_name.numpy())
 print("Age:", data['age'].numpy())
 print("Age(scaled):", data['age_scaled'].numpy() )
!wget -q https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip
!unzip -o uncased_L-12_H-768_A-12.zip<define_variables>
%%time
validation_dataset = load_dataset(VAL_FNAMES['fold_2'])
 print("Examples of the validation data:")
 for image, label, data, label_name in validation_dataset.take(1):
print("The image batch size:", image.numpy().shape)
 print("Label:", label.numpy())
 print("Label name:", label_name.numpy())
 print("Age:", data['age'].numpy())
 print("Age(scaled):", data['age_scaled'].numpy() )
SEQ_LEN = 128
LR = 1e-4
pretrained_path = 'uncased_L-12_H-768_A-12'
config_path = os.path.join(pretrained_path, 'bert_config.json')
 checkpoint_path = os.path.join(pretrained_path, 'bert_model.ckpt')
 vocab_path = os.path.join(pretrained_path, 'vocab.txt')
<feature_engineering>
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FNAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
token_dict = {}
with codecs.open(vocab_path, 'r', 'utf8')as reader:
for line in reader:
token = line.strip()
token_dict[token] = len(token_dict)
 tokenizer = Tokenizer(token_dict )<load_from_csv>
np.set_printoptions(threshold=15, linewidth=80)
 def batch_to_numpy_images_and_labels(databatch):
if len(databatch)==4:
images, labels, _, _ = databatch
numpy_images = images.numpy()
numpy_labels = labels.numpy()
else:
images, _ = databatch
numpy_images = images.numpy()
numpy_labels = [None for _ in enumerate(numpy_images)]
return numpy_images, numpy_labels
def title_from_label_and_target(label, correct_label):
if correct_label is None:
return CLASSES[label], True
correct =(label == correct_label)
 return "{} [{}{}{}]".format(CLASSES[label], 'OK' if correct else 'NO', u"\u2192"
if not correct else '',
CLASSES[correct_label] if not correct else ''), correct
def display_one_image(image, title, subplot, red=False, titlesize=16):
plt.subplot(*subplot)
 plt.axis('off')
 plt.imshow(image)
 if len(title)> 0:
plt.title(title, fontsize=int(titlesize)if not red else int(titlesize/1.2),
color='red' if red else 'black', fontdict={'verticalalignment':'center'},
pad=int(titlesize/1.5)
)
 return(subplot[0], subplot[1], subplot[2]+1)
 def display_batch_of_images(databatch, predictions=None):

 images, labels = batch_to_numpy_images_and_labels(databatch)
 if labels is None:
labels = [None for _ in enumerate(images)]
rows = int(math.sqrt(len(images)))
 cols = len(images)//rows
FIGSIZE = 13.0
SPACING = 0.1
subplot=(rows,cols,1)
 if rows < cols:
plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))
else:
plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))
for i,(image, label)in enumerate(zip(images[:rows*cols], labels[:rows*cols])) :
title = '' if label is None else CLASSES[label]
correct = True
if predictions is not None:
title, correct = title_from_label_and_target(predictions[i], label)
 dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3
subplot = display_one_image(image, title, subplot,
not correct, titlesize=dynamic_titlesize)
 plt.tight_layout()
if label is None and predictions is None:
plt.subplots_adjust(wspace=0, hspace=0)
 else:
plt.subplots_adjust(wspace=SPACING, hspace=SPACING)
 plt.show()
data = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 data_eval = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 data.head()<categorify>
training_dataset = training_dataset.batch(20)
 train_batch = iter(training_dataset )
def remove_URL(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'',text)
 def remove_html(text):
html=re.compile(r'<.*?>')
 return html.sub(r'',text)
 def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 def remove_punct(text):
table=str.maketrans('','',string.punctuation)
 return text.translate(table)
 def remove_tag(text):
url = re.compile(r'(?<=\@)(.*? )(?=)')
 return url.sub(r'',text)
 def remove_space(text):
return text.replace(r'%20',' ')
 def cleanse_data(data_in):
data = data_in.copy()
data['text'] = data['text'].apply(lambda x:remove_URL(x))
data['text'] = data['text'].apply(lambda x:remove_html(x))
data['text'] = data['text'].apply(lambda x:remove_emoji(x))
data['text'] = data['text'].apply(lambda x:remove_tag(x))
data['text'] = data['text'].apply(lambda x:remove_punct(x))
data['keyword'].fillna('Nothing', inplace=True)
 data['keyword'] = data['keyword'].apply(lambda x:remove_space(x))
data['keyword'] = data['keyword'].apply(lambda x:remove_punct(x))
data['location'].fillna('Nothing', inplace=True)
 data['location'] = data['location'].apply(lambda x:remove_punct(x))
return data 
<drop_column>
display_batch_of_images(next(train_batch))
data_cleansed = cleanse_data(data )<prepare_x_and_y>
del training_dataset, train_batch
gc.collect()
def get_X(data, column_name = 'text'):
X1 = [tokenizer.encode(text, max_len=SEQ_LEN)[0] for text in data[column_name] ]
X1 = np.array(X1)
 X2 = np.zeros_like(X1)
 X = [X1,X2]
return X
X_train = get_X(data_cleansed)
 y_train = np.array(data_cleansed.target )<train_model>
LR_START = 0.000005
LR_MAX = 0.00000725 * strategy.num_replicas_in_sync
LR_MIN = 0.000005
LR_RAMPUP_EPOCHS = 5
LR_SUSTAIN_EPOCHS = 4
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)
 rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))
bert_model = load_trained_model_from_checkpoint(
 config_path,
checkpoint_path,
training=True,
trainable=True,
seq_len=SEQ_LEN,
)<feature_engineering>
train['target'].values
class_weights = class_weight.compute_class_weight(class_weight='balanced',
classes=np.unique(train['target'].values),
y=train['target'].values, 
)
 class_weights = {i : class_weights[i] for i in range(len(class_weights)) }
print(class_weights )
sess = K.get_session()
uninitialized_variables = set([i.decode('ascii')for i in sess.run(tf.report_uninitialized_variables())])
 init_op = tf.variables_initializer(
 [v for v in tf.global_variables() if v.name.split(':')[0] in uninitialized_variables] 
)
 sess.run(init_op )<choose_model_class>
tab_feats=['age_scaled',
'sex_female',
'sex_male',
'sex_unknown',
'site_head/neck',
'site_lower extremity',
'site_oral/genital',
'site_palms/soles',
'site_torso',
'site_unknown',
'site_upper extremity'
]
N_TAB_FEATS=len(tab_feats)
 print(f"The number of tabular features is {N_TAB_FEATS}." )
def get_model(neuron=256):
inputs = bert_model.inputs[:2]
outputs = bert_model.get_layer('NSP-Dense' ).output
outputs = keras.layers.Dense(neuron, activation='relu' )(outputs)
 outputs = keras.layers.Dense(units=1, activation='sigmoid' )(outputs)
 model = keras.models.Model(inputs, outputs)
 for layer in model.layers[:-20]:
layer.trainable = False
model.compile(RAdam(learning_rate =LR),loss='binary_crossentropy',metrics=['accuracy'])
 return model<save_to_csv>
%time
def get_model() :
with strategy.scope() :
pretrained_model = efn.EfficientNetB3(input_shape=(*IMAGE_SIZE, 3),
weights='imagenet',
include_top=False 
)
 pretrained_model.trainable = True
inp1 = tf.keras.layers.Input(shape=(*IMAGE_SIZE, 3), name='inp1')
 inp2 = tf.keras.layers.Input(shape=(N_TAB_FEATS), name='inp2')
 x=pretrained_model(inp1)
 x=tf.keras.layers.GlobalAveragePooling2D()(x)
 x=tf.keras.layers.Dense(512,
kernel_regularizer=tf.keras.regularizers.l2(l=0.01),
activation='relu' )(x)
 x=tf.keras.layers.Dropout(0.2 )(x)
 x=tf.keras.layers.Dense(256,
kernel_regularizer=tf.keras.regularizers.l2(l=0.01),
activation='relu' )(x)
 x=tf.keras.layers.Dropout(0.2 )(x)
 x=tf.keras.layers.Dense(128,
kernel_regularizer=tf.keras.regularizers.l2(l=0.01),
activation='relu' )(x)
 x=tf.keras.layers.Dropout(0.2 )(x)
 x=tf.keras.layers.Dense(64, kernel_regularizer=tf.keras.regularizers.l2(l=0.01),
activation='relu' )(x)
 x=tf.keras.layers.Dropout(0.2 )(x)
 y=tf.keras.layers.Dense(100,
kernel_regularizer=tf.keras.regularizers.l2(l=0.01),
activation='relu' )(inp2)
 concat=tf.keras.layers.concatenate([y, x])
 output = tf.keras.layers.Dense(1, activation='sigmoid', name='output' )(concat)
 model = tf.keras.models.Model(inputs=[inp1,inp2], outputs=[output])
 model.compile(
 optimizer='adam',
loss = 'binary_crossentropy',
metrics=[tf.keras.metrics.AUC() ], 
)
 return model
<train_model>
del model
gc.collect()
model = get_model(neuron=32)
 model.summary()
history = model.fit(X_train,y_train,
epochs=4,
batch_size=64,
verbose=1,
validation_split=0.1 )<predict_on_test>
if colab:
SAVE_FOLDER=NAME
if tstamp:
time_zone = pytz.timezone('America/Chicago')
 current_datetime = datetime.now(time_zone)
 ts=current_datetime.strftime("%m%d%H%M%S")
 SAVE_FOLDER+='_'+ts
SAVE_FOLDER=PATH/SAVE_FOLDER
if not os.path.exists(SAVE_FOLDER):
os.mkdir(SAVE_FOLDER)
 else:
SAVE_FOLDER=Path('/kaggle/working' )
y_pred = model.predict(X_train, verbose=True ).flatten()
y_pred = np.round(y_pred ).astype(int ).reshape(X_train[0].shape[0])
 pred = pd.DataFrame(data=y_pred, columns=['pred'])
 pred_df = pd.concat([data['text'], data_cleansed['text'], data['target'],pred], axis=1)
 incorrect = pred_df[pred_df.pred!=pred_df.target]<prepare_x_and_y>
def setup_input(image, label, data, label_name):
tab_data=[tf.cast(data[tfeat], dtype=tf.float32)for tfeat in tab_feats]
tabular=tf.stack(tab_data)
 return {'inp1': image, 'inp2': tabular}, label
data_eval_cleansed = cleanse_data(data_eval)
 X_eval = get_X(data_eval_cleansed )<predict_on_test>
def data_augment(data, label):
data['inp1'] = tf.image.random_flip_left_right(data['inp1'])
 data['inp1'] = tf.image.random_flip_up_down(data['inp1'])
 return data, label
y_eval = model.predict(X_eval, verbose=True)
 y_eval = y_eval.flatten()
y_eval = np.round(y_eval ).astype(int ).reshape(X_eval[0].shape[0] )<create_dataframe>
def get_training_dataset(dataset):
dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
pred = pd.DataFrame(data=y_eval, columns=['pred'])
 pd.concat([data_eval, data_eval_cleansed['text'], pred], axis=1 )<save_to_csv>
def get_validation_dataset(dataset):
dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
output = pd.DataFrame({'id': data_eval_cleansed.id, 'target': y_eval})
 output.to_csv('my_submission_20200205.csv', index=False)
 print("Your submission was successfully saved!" )<install_modules>
def setup_test_image(image, data):
tab_data=[tf.cast(data[tfeat], dtype=tf.float32)for tfeat in tab_feats]
tabular=tf.stack(tab_data)
 return {'inp1': image, 'inp2': tabular}
def setup_test_name(image, data):
return data['image_name']
def get_test_dataset(dataset):
dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def average_predictions(X, fn):
y_probas=[]
for idx in range(NBEST):
tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 gc.collect()
print(f"Predicting: fold {fn}, model {idx+1} out of {NBEST}...")
 with strategy.scope() :
path_best_model=f'best_model_fold_{fn}_{idx}.hdf5'
TRAINED_MODEL_FOLDER=Path('.. /input/effb3512/EffNB3_512_0702063710')
 model=tf.keras.models.load_model(TRAINED_MODEL_FOLDER/path_best_model)
 y=model.predict(X)
 y = rankdata(y)/len(y)
 y_probas.append(y)
 y_probas=np.average(y_probas, axis=0)
 return y_probas
!pip install tweet-preprocessor<import_modules>
%%time
preds = pd.DataFrame({'image_name': np.zeros(len(test)) , 'target': np.zeros(len(test)) })
 test_ds = load_dataset(TEST_FNAMES, labeled=False, ordered=True)
 test_images_ds = test_ds.map(setup_test_image, num_parallel_calls=AUTO)
 test_images_ds = get_test_dataset(test_images_ds)
 test_ds = get_test_dataset(test_ds)
 test_ids_ds = test_ds.map(setup_test_name, num_parallel_calls=AUTO ).unbatch()
preds['image_name'] = next(iter(test_ids_ds.batch(N_TEST_IMGS)) ).numpy().astype('U')
 preds['target'] = np.average([average_predictions(test_images_ds, fn)for fn in range(1, NFOLDS+1)], axis = 0 )
import os
from google.cloud import storage, automl_v1beta1 as automl
import numpy as np
import pandas as pd
from sklearn import feature_extraction, linear_model, model_selection, preprocessing
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
import scipy as sp
from sklearn.model_selection import train_test_split
from sklearn.model_selection import StratifiedKFold, KFold, GridSearchCV
from sklearn.svm import SVC
from sklearn.metrics import accuracy_score, confusion_matrix, f1_score
from sklearn.naive_bayes import GaussianNB, MultinomialNB
import seaborn as sns
import matplotlib.pyplot as plt
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
from nltk.stem import WordNetLemmatizer
import nltk
from collections import Counter
import string
import re
from nltk.corpus import wordnet as wn
from statistics import mean
import preprocessor<import_modules>
del sub['target']
sub = sub.merge(preds, on='image_name')
 sub.head()
from sklearn import metrics<string_transform>
print(f"The lengths of the submission file and `test` are {len(sub)} and {len(test)}, respectively.")
 print(f"The number of NA's in the submission file is {sub.isna().sum().sum() }." )
<load_from_csv><EOS> 
if colab:
OUT_FOLDER=SAVE_FOLDER
else:
OUT_FOLDER=Path('')
 sub.to_csv(OUT_FOLDER/'submission.csv', index=False )