Kaggle competition: What's Cooking? (Kernels Only)
NAME='EffNB6_512'
NFOLDS=5
NBEST=1
SEED=311
ef = 3
DROPOUT = True
droprate=0.5
dropct= 4
dropsize=0.2
TTA = 3
EPOCHS = 10
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
IMAGE_SIZE = [384,384]
ANATOM = 1
dim = IMAGE_SIZE[0]
DIM = dim
AUTO = tf.data.experimental.AUTOTUNE
k = IMAGE_SIZE[0]
GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-%ix%i'%(k,k))
GCS_PATH2 = KaggleDatasets().get_gcs_path('isic2019-%ix%i'%(k,k))
GCS_PATH3 = KaggleDatasets().get_gcs_path('malignant-v2-%ix%i'%(k,k))
USE_EXT_DATA = True
files_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')))
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')) )<load_from_csv>
train_df = pd.read_json(".. /input/train.json" ).set_index("id")
 train_df.cuisine = train_df.cuisine.astype("category")
 train_df.head()
def append_path(pre):
return np.vectorize(lambda file: os.path.join(GCS_DS_PATH, pre, file))
sub = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')
 train = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/train.csv')
 sns.countplot(train['target'] )<define_variables>
cuisines = train_df.cuisine.cat.categories.values.tolist()
texts = []
labels = []
label2index = { cuisine: i for i, cuisine in enumerate(cuisines)}
for i, row in train_df.iterrows() :
texts.append("
".join(row.ingredients))
labels.append(label2index[row.cuisine])
 labels = to_categorical(np.asarray(labels, dtype=np.int32))
word_count = Counter()
def lemmatize(texts):
global word_count
wnl = nltk.WordNetLemmatizer()
for text in texts:
tokens_recipe = []
for sentence in text.split("
"):
tokens_ingredient = [ wnl.lemmatize(w)for w in wordpunct_tokenize(sentence.lower())if w.isalpha() ]
word_count.update(tokens_ingredient)
 tokens_recipe.append(" ".join(tokens_ingredient))
yield " ".join(tokens_recipe)
 def preprocess(texts):
processed_texts = list(lemmatize(texts))
black_list = [ word for word, count in word_count.items() if count < 5 ]
return [[ word for word in sentence.split() if word not in black_list] for sentence in processed_texts ]
tokenizer = Tokenizer(oov_token="<UNK>")
 processed_texts = preprocess(texts)
 tokenizer.fit_on_texts(processed_texts)
 feature_matrix = tokenizer.texts_to_matrix(processed_texts)
 word2index = tokenizer.word_index
print("Unique tokens: {}".format(len(word2index)))
 feature_matrix.shape
%%time
ALL_TRAIN=tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')
 ALL_TRAIN1 = tf.io.gfile.glob(GCS_PATH2 + '/train*.tfrec')
 ALL_TRAIN2 = tf.io.gfile.glob(GCS_PATH3 + '/train*.tfrec')
 ALL_TRAIN<feature_engineering>
def build_model(hidden_units, dropout):
model = Sequential()
model.add(Dense(hidden_units, input_shape=[1722,], activation="relu", name="hidden"))
model.add(Dropout(dropout, name="dropout"))
model.add(Dense(20, name="output"))
model.compile("adam", "categorical_hinge", metrics=["accuracy"])
 return model
tot_train = len(ALL_TRAIN)
 skip = tot_train//NFOLDS
VAL_FNAMES={}
x = 0
for n in range(1, NFOLDS+1):
VAL_FNAMES[f"fold_{n}"] = ALL_TRAIN[x : x+skip]
x += skip
if USE_EXT_DATA :
ALL_TRAIN += ALL_TRAIN1
ALL_TRAIN += ALL_TRAIN2
TRAIN_FNAMES={f'fold_{i}': list(set(ALL_TRAIN)-set(VAL_FNAMES[f'fold_{i}']))
for i in range(1, NFOLDS+1)}
TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test*.tfrec' )<count_values>
X_train, X_val, y_train, y_val = train_test_split(feature_matrix, labels, test_size=0.2, random_state=42 )
print(f"total folds : {len(TRAIN_FNAMES)} Total per fold : {len(TRAIN_FNAMES['fold_1'])}" )<define_variables>
test_df = pd.read_json(".. /input/test.json")
 test_texts = []
for i, row in test_df.iterrows() :
test_texts.append("
".join(row.ingredients))
processed_texts = preprocess(test_texts)
 test_feature_matrix = tokenizer.texts_to_matrix(processed_texts)
 test_feature_matrix.shape
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed )<load_from_csv>
kf = KFold(n_splits=5, shuffle=True)
 models = []
early_stopping = EarlyStopping(monitor='val_loss', patience=5, verbose=1)
 oos_y = []
oos_pred = []
for i,(train_index, test_index)in enumerate(kf.split(feature_matrix), start=1):
X_train, y_train = feature_matrix[train_index], labels[train_index]
X_val, y_val = feature_matrix[test_index], labels[test_index]
model = build_model(2048, 0.8)
 chk_point = ModelCheckpoint("best-model-{}.h5".format(i), monitor='val_loss', save_best_only=True, save_weights_only=True)
 model.fit(X_train, y_train,
validation_data=(X_val, y_val),
epochs=50,
callbacks=[early_stopping, chk_point],
verbose=0,
batch_size=128)
 model.load_weights("best-model-{}.h5".format(i))
y_pred = model.predict(X_val)
 loss = hinge_loss(y_val.argmax(axis=1), y_pred)
 accuracy = accuracy_score(y_val.argmax(axis=1), y_pred.argmax(axis=1))
oos_y.append(y_val)
 oos_pred.append(y_pred)
 models.append(model)
 print("Fold {}, loss: {}, accuracy: {:.2%}".format(i, loss, accuracy))
y_true = np.vstack(oos_y)
 y_pred = np.vstack(oos_pred)
 loss = hinge_loss(y_true.argmax(axis=1), y_pred)
 accuracy = accuracy_score(y_true.argmax(axis=1), y_pred.argmax(axis=1))
print("loss: {}, accuracy: {:.2%}".format(loss, accuracy))
if colab:
PATH=Path('/content/gdrive/My Drive/kaggle/input/siim-isic-melanoma-classification/')
 train=pd.read_csv(PATH/'train.csv.zip')
 else:
PATH=Path('/kaggle/input/siim-isic-melanoma-classification/')
 train=pd.read_csv(PATH/'train.csv')
 test=pd.read_csv(PATH/'test.csv')
 sub=pd.read_csv(PATH/'sample_submission.csv')
 seed_everything(SEED )<set_options>
summary = np.zeros(( 20, 20), dtype=np.int32)
 for y_true_i, y_pred_i in zip(y_true.argmax(axis=1), y_pred.argmax(axis=1)) :
summary[y_true_i, y_pred_i] += 1
summary_df = pd.DataFrame(summary,
columns=cuisines,
index=cuisines)
 summary_df
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<categorify>
test_pred = np.zeros(( 9944, 5), dtype=np.int32)
 for i, model in enumerate(models):
y_pred = model.predict(test_feature_matrix)
 test_pred[:, i] = y_pred.argmax(axis=1)
 def voting(arr):
return np.bincount(arr ).argmax()
predictions = np.apply_along_axis(voting, 1, test_pred )
<train_model><EOS> 
result = pd.Series(pd.Categorical.from_codes(predictions, cuisines), test_df.id, name="cuisine")
 result.to_csv("submission.csv", header=True)
 result.value_counts()