Kaggle competition: What's Cooking? (Kernels Only)
<SOS> metric: CategorizationAccuracy Kaggle data source: whats-cooking?-(kernels-only)<filter>
nltk.download('wordnet' )
print(len(df_train))
print(len(df_train[df_train.target==1])) 
<set_options>
def porter_stem_ingredients(ingredient):
porter_stemmer = PorterStemmer()
tokens = ingredient.split()
stemmed_tokens = [porter_stemmer.stem(token)for token in tokens]
ingredient = ' '.join(stemmed_tokens)
 return ingredient
DEVICE = "TPU"
bs = 16
CFG = dict(
 net_count = 7,
batch_size = bs,
read_size = 512,
crop_size = 352,
net_size = 352,
LR_START = 0.000005,
LR_MAX = 0.000020,
LR_MIN = 0.000001,
LR_RAMPUP_EPOCHS = 5,
LR_SUSTAIN_EPOCHS = 0,
LR_EXP_DECAY = 0.8,
epochs = 12,
rot = 180.0,
shr = 2.0,
hzoom = 8.0,
wzoom = 8.0,
hshift = 8.0,
wshift = 8.0,
optimizer = 'adam',
label_smooth_fac = 0.05,
tta_steps = 25
)<choose_model_class>
def snowball_stem_ingredients(ingredient):
snowball_stemmer = SnowballStemmer('english')
 tokens = ingredient.split()
stemmed_tokens = [snowball_stemmer.stem(token)for token in tokens]
ingredient = ' '.join(stemmed_tokens)
 return ingredient
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )<normalization>
def wordnet_lemmatizer_ingredients(ingredient):
lemmatizer = WordNetLemmatizer()
tokens = ingredient.split()
lemmatized_tokens = [lemmatizer.lemmatize(token)for token in tokens]
ingredient = ' '.join(lemmatized_tokens)
 return ingredient
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, cfg):
DIM = cfg["read_size"]
XDIM = DIM%2
rot = cfg['rot'] * tf.random.normal([1], dtype='float32')
 shr = cfg['shr'] * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ cfg['hzoom']
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ cfg['wzoom']
h_shift = cfg['hshift'] * tf.random.normal([1], dtype='float32')
 w_shift = cfg['wshift'] * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )<prepare_x_and_y>
def main_preprocessing(ingredient):
ingredient = char_preprocess_ingredients(ingredient)
 ingredient = wordnet_lemmatizer_ingredients(ingredient)
 ingredient = snowball_stem_ingredients(ingredient)
 ingredient = porter_stem_ingredients(ingredient)
 return ingredient
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'diagnosis' : tf.io.FixedLenFeature([], tf.int64),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['target']
def read_unlabeled_tfrecord(example, return_image_name):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['image_name'] if return_image_name else 0
def prepare_image(img, cfg=None, augment=True):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.image.resize(img, [cfg['read_size'], cfg['read_size']])
 img = tf.cast(img, tf.float32)/ 255.0
if augment:
img = transform(img, cfg)
 img = tf.image.random_crop(img, [cfg['crop_size'], cfg['crop_size'], 3])
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_hue(img, 0.01)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 else:
img = tf.image.central_crop(img, cfg['crop_size'] / cfg['read_size'])
 img = tf.image.resize(img, [cfg['net_size'], cfg['net_size']])
 img = tf.reshape(img, [cfg['net_size'], cfg['net_size'], 3])
 return img
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )<categorify>
warnings.filterwarnings('ignore')
 df_train = pd.read_json('.. /input/train.json')
 df_train['ingredients'] = df_train['ingredients'].apply(lambda x : [main_preprocessing(y)for y in x])
 df_train['all_ingredients'] = df_train['ingredients'].map(";".join )
def get_dataset(files, cfg, augment = False, shuffle = False, repeat = False,
labeled=True, return_image_names=True):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTO)
 ds = ds.map(lambda img, imgname_or_label:(prepare_image(img, augment=augment, cfg=cfg),
imgname_or_label),
num_parallel_calls=AUTO)
 ds = ds.batch(cfg['batch_size'] * REPLICAS)
 ds = ds.prefetch(AUTO)
 return ds<prepare_x_and_y>
vectorizer = CountVectorizer(analyzer = "word",
ngram_range =(1,1),
binary = True,
tokenizer = None,
preprocessor = None,
stop_words = None,
max_df = 0.99)
 X = vectorizer.fit_transform(df_train['all_ingredients'].values )
def show_dataset(thumb_size, cols, rows, ds):
mosaic = PIL.Image.new(mode='RGB', size=(thumb_size*cols +(cols-1),
thumb_size*rows +(rows-1)))
 for idx, data in enumerate(iter(ds)) :
img, target_or_imgid = data
ix = idx % cols
iy = idx // cols
img = np.clip(img.numpy() * 255, 0, 255 ).astype(np.uint8)
 img = PIL.Image.fromarray(img)
 img = img.resize(( thumb_size, thumb_size), resample=PIL.Image.BILINEAR)
 mosaic.paste(img,(ix*thumb_size + ix,
iy*thumb_size + iy))
display(mosaic)
<create_dataframe>
enc = LabelEncoder()
y = enc.fit_transform(df_train.cuisine )
ds = get_dataset(files_train, CFG ).unbatch().take(12*5)
<create_dataframe>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2 )
ds = tf.data.TFRecordDataset(files_train, num_parallel_reads=AUTO)
 ds = ds.take(1 ).cache().repeat()
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 ds = ds.map(lambda img, target:(prepare_image(img, cfg=CFG, augment=True), target),
num_parallel_calls=AUTO)
 ds = ds.take(12*5)
 ds = ds.prefetch(AUTO)
<choose_model_class>
logistic = LogisticRegression(C=1, class_weight=None, dual=False, fit_intercept=True,
intercept_scaling=1, max_iter=100, multi_class='warn',
n_jobs=None, penalty='l2', random_state=1, solver='warn',
tol=0.1, verbose=0, warm_start=False)
 logistic.fit(X_train, y_train)
 print('Accuracy: %.5f' % logistic.score(X_test, y_test))
def get_lr_callback(cfg):
lr_start = cfg['LR_START']
lr_max = cfg['LR_MAX'] * strategy.num_replicas_in_sync
lr_min = cfg['LR_MIN']
lr_ramp_ep = cfg['LR_RAMPUP_EPOCHS']
lr_sus_ep = cfg['LR_SUSTAIN_EPOCHS']
lr_decay = cfg['LR_EXP_DECAY']
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback<choose_model_class>
ovrLR = OneVsRestClassifier(estimator=logistic, n_jobs=1)
 ovrLR.fit(X_train, y_train)
 print('Accuracy: %.5f' % ovrLR.score(X_test, y_test))
def get_model(cfg):
model_input = tf.keras.Input(shape=(cfg['net_size'], cfg['net_size'], 3), name='imgIn')
 dummy = tf.keras.layers.Lambda(lambda x:x )(model_input)
 constructor = getattr(efn, f'EfficientNetB6')
 x = constructor(include_top=False, weights='imagenet',
input_shape=(cfg['net_size'], cfg['net_size'], 3),
pooling='avg' )(dummy)
 x = tf.keras.layers.Dense(1, activation='sigmoid' )(x)
 outputs = [x]
model = tf.keras.Model(model_input, outputs, name='aNetwork')
 model.summary()
return model<choose_model_class>
lsvc = LinearSVC(C=0.2, class_weight=None, dual=True, fit_intercept=True,
intercept_scaling=1, loss='squared_hinge', max_iter=100,
multi_class='ovr', penalty='l2', random_state=1, tol=0.0001,
verbose=0)
 lsvc.fit(X_train, y_train)
 print('Accuracy: %.5f' % lsvc.score(X_test, y_test))
def compile_new_model(cfg):
with strategy.scope() :
model = get_model(cfg)
 losses = [tf.keras.losses.BinaryCrossentropy(label_smoothing = cfg['label_smooth_fac'])]
model.compile(
 optimizer = cfg['optimizer'],
loss = losses,
metrics = [tf.keras.metrics.AUC(name='auc')])
 return model<choose_model_class>
ovrLSVC = OneVsRestClassifier(estimator=lsvc, n_jobs=1)
 ovrLSVC.fit(X_train, y_train)
 print('Accuracy: %.5f' % ovrLSVC.score(X_test, y_test))
nsplits = 5
rand = 1023
folds = KFold(n_splits=nsplits, shuffle = True, random_state = rand )<create_dataframe>
etc = ExtraTreesClassifier(bootstrap=False, class_weight=None, criterion='gini',
max_depth=None, max_features=0.25, max_leaf_nodes=None,
min_impurity_decrease=0.0, min_impurity_split=None,
min_samples_leaf=1, min_samples_split=2,
min_weight_fraction_leaf=0.0, n_estimators=400, n_jobs=None,
oob_score=False, random_state=1, verbose=0, warm_start=False)
 etc.fit(X_train, y_train)
 print('Accuracy: %.5f' % etc.score(X_test, y_test))
pred_tr = pd.DataFrame()
cnt = 0
for tr_idx,va_idx in folds.split(files_train):
CFG['batch_size'] = bs
files_train_tr = files_train[tr_idx]
files_train_va = files_train[va_idx]
ds_train = get_dataset(files_train_tr, CFG, augment=True, shuffle=True, repeat=True)
 ds_train = ds_train.map(lambda img, label:(img, tuple([label])))
 steps_train = count_data_items(files_train_tr)/(CFG['batch_size'] * REPLICAS)
 model = compile_new_model(CFG)
 history = model.fit(ds_train,
verbose = 1,
steps_per_epoch = steps_train,
epochs = CFG['epochs'],
callbacks = [get_lr_callback(CFG)])
 CFG['batch_size'] = 256
cnt_train = count_data_items(files_train_va)
 steps = cnt_train /(CFG['batch_size'] * REPLICAS)* CFG['tta_steps']
ds_trainAug = get_dataset(files_train_va, CFG, augment=True, repeat=True,
labeled=False, return_image_names=False)
 probs = model.predict(ds_trainAug, verbose=1, steps=steps)
 probs = probs[:cnt_train * CFG['tta_steps'],:]
probs = np.stack(np.split(probs, CFG['tta_steps'], axis=0), axis=0)
 ds = get_dataset(files_train_va, CFG, augment=False, repeat=False,
labeled=False, return_image_names=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())])
 pred = pd.DataFrame(dict(
 image_name = image_names,
target = np.mean(probs[:,:,0], axis=0)))
 pred_tr = pd.concat([pred_tr, pred], axis=0)
 cnt_test = count_data_items(files_test)
 steps = cnt_test /(CFG['batch_size'] * REPLICAS)* CFG['tta_steps']
ds_testAug = get_dataset(files_test, CFG, augment=True, repeat=True,
labeled=False, return_image_names=False)
 probs = model.predict(ds_testAug, verbose=1, steps=steps)
 probs = probs[:cnt_test * CFG['tta_steps'],:]
probs = np.stack(np.split(probs, CFG['tta_steps'], axis=0), axis=0)
 if cnt == 0:
probs_sub = probs/nsplits
cnt = 1
else:
probs_sub += probs/nsplits<save_to_csv>
ovrETC = OneVsRestClassifier(estimator=etc, n_jobs=1)
 ovrETC.fit(X_train, y_train)
 print('Accuracy: %.5f' % ovrETC.score(X_test, y_test))
pred_tr = pred_tr.sort_values('image_name')
 pred_tr.to_csv('pred_tr.csv', index=False )<merge>
mlp = MLPClassifier(activation='relu', alpha=0.0001, batch_size='auto', beta_1=0.9,
beta_2=0.999, early_stopping=False, epsilon=1e-08,
hidden_layer_sizes=(100, 100), learning_rate='constant',
learning_rate_init=0.001, max_iter=200, momentum=0.9,
n_iter_no_change=10, nesterovs_momentum=True, power_t=0.5,
random_state=1, shuffle=True, solver='adam', tol=0.0001,
validation_fraction=0.1, verbose=False, warm_start=False)
 mlp.fit(X_train, y_train)
 print('Accuracy: %.5f' % mlp.score(X_test, y_test))
pred_tr = pred_tr.merge(df_train, on = ["image_name"], how = "left")
 print(roc_auc_score(pred_tr["target_y"], pred_tr["target_x"]))<create_dataframe>
ovrMLP = OneVsRestClassifier(estimator=mlp, n_jobs=1)
 ovrMLP.fit(X_train, y_train)
 print('Accuracy: %.5f' % ovrMLP.score(X_test, y_test))
ds = get_dataset(files_test, CFG, augment=False, repeat=False,
labeled=False, return_image_names=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())] )<save_to_csv>
vc = VotingClassifier(n_jobs=1, estimators=[ 
('clf1', ovrMLP), 
('clf2', ovrETC), 
('clf3', ovrLSVC), 
('clf4', ovrLR)
 ])
 vc.fit(X_train, y_train)
 print('Accuracy: %.5f' % vc.score(X_test, y_test))
submission = pd.DataFrame(dict(
 image_name = image_names,
target = np.mean(probs_sub[:,:,0], axis=0)))
 submission = submission.sort_values('image_name')
 submission.to_csv('submission.csv', index=False )<import_modules>
df_test = pd.read_json('.. /input/test.json')
 df_test['ingredients'] = df_test['ingredients'].apply(lambda x : [main_preprocessing(y)for y in x])
 df_test['all_ingredients'] = df_test['ingredients'].map(";".join )
import ast
import numba
import string
import numpy as np
import pandas as pd
from itertools import combinations, permutations, product
from math import sqrt, factorial
from sklearn.neighbors import KDTree
from sympy import isprime, primerange
from tqdm import tqdm_notebook<define_variables>
X = vectorizer.transform(df_test['all_ingredients'].values )
K = 4<statistical_test>
y_pred = vc.predict(X )
<feature_engineering><EOS> 
y_pred = enc.inverse_transform(y_pred)
 y_pred = pd.DataFrame({'cuisine' : y_pred , 'id' : df_test.id }, columns=['id', 'cuisine'])
 y_pred.to_csv('submission.csv', index = False )