Kaggle competition: Instacart Market Basket Analysis
<SOS> metric: MeanFScore Kaggle data source: instacart-market-basket-analysis<load_pretrained>
gc.enable()
model1.load_weights('.. /input/kerasefficientnetb3/Train1_model_59.h5')
 model2.load_weights('.. /input/kerasefficientnetb3/Train1_model_64.h5')
 model3.load_weights('.. /input/kerasefficientnetb3/Train1_model_68.h5')
 model4.load_weights('.. /input/kerasefficientnetb3/Train1_model_57.h5')
 model5.load_weights('.. /input/kerasefficientnetb3/Train1_model_70.h5' )<define_variables>
orders = pd.read_csv('.. /input/orders.csv')
 order_products_train = pd.read_csv('.. /input/order_products__train.csv')
 order_products_prior = pd.read_csv('.. /input/order_products__prior.csv')
 products = pd.read_csv('.. /input/products.csv')
 aisles = pd.read_csv('.. /input/aisles.csv')
 departments = pd.read_csv('.. /input/departments.csv' )
class TestDataGenerator(keras.utils.Sequence):
def __init__(self, X, batch_size = 16, img_size =(512, 512, 3), *args, **kwargs):
self.X = X
self.indices = np.arange(len(self.X))
self.batch_size = batch_size
self.img_size = img_size
def __len__(self):
return int(ceil(len(self.X)/ self.batch_size))
def __getitem__(self, index):
indices = self.indices[index*self.batch_size:(index+1)*self.batch_size]
X = self.__data_generation(indices)
 return X
def __data_generation(self, indices):
X = np.empty(( self.batch_size, *self.img_size))
for i, index in enumerate(indices):
image = self.X[index]
image = np.stack(( image,)*CHANNELS, axis=-1)
 image = image.reshape(-1, HEIGHT_NEW, WIDTH_NEW, CHANNELS)
 X[i,] = image
return X<define_variables>
aisles['aisle'] = aisles['aisle'].astype('category')
 departments['department'] = departments['department'].astype('category')
 orders['eval_set'] = orders['eval_set'].astype('category')
 products['product_name'] = products['product_name'].astype('category' )
tgt_cols = ['grapheme_root','vowel_diacritic','consonant_diacritic']
row_ids, targets = [], []
for i in range(0, 4):
test_files = []
df = pd.read_parquet(os.path.join(DIR, 'test_image_data_'+str(i)+'.parquet'))
image_ids = df['image_id'].values
df = df.drop(['image_id'], axis = 1)
 X = []
for image_id, index in zip(image_ids, range(df.shape[0])) :
test_files.append(image_id)
 X.append(resize_image(df.loc[df.index[index]].values, WIDTH_NEW, HEIGHT_NEW))
data_generator_test = TestDataGenerator(X, batch_size = BATCH_SIZE, img_size =(HEIGHT_NEW, WIDTH_NEW, CHANNELS))
preds1 = model1.predict_generator(data_generator_test, verbose = 1)
 preds2 = model2.predict_generator(data_generator_test, verbose = 1)
 preds3 = model3.predict_generator(data_generator_test, verbose = 1)
 preds4 = model4.predict_generator(data_generator_test, verbose = 1)
 preds5 = model5.predict_generator(data_generator_test, verbose = 1)
 for i, image_id in zip(range(len(test_files)) , test_files):
for subi, col in zip(range(len(preds1)) , tgt_cols):
sub_preds1 = preds1[subi]
sub_preds2 = preds2[subi]
sub_preds3 = preds3[subi]
sub_preds4 = preds4[subi]
sub_preds5 = preds5[subi]
row_ids.append(str(image_id)+'_'+col)
 sub_pred_value = np.argmax(( sub_preds1[i] + sub_preds2[i] + sub_preds3[i] + sub_preds4[i] + sub_preds5[i])/ 5)
 targets.append(sub_pred_value)
 del df
gc.collect()<save_to_csv>
op = orders.merge(order_products_prior, on='order_id', how='inner')
 op.head()
submit_df = pd.DataFrame({'row_id':row_ids,'target':targets}, columns = ['row_id','target'])
 submit_df.to_csv('submission.csv', index = False)
 print(submit_df.head(40))<load_from_csv>
user = op.groupby('user_id')['order_number'].max().to_frame('u_total_orders')
 add1 = op.groupby(['user_id','product_id'])['product_id'].count().to_frame('times_p_order')
 add1 = add1.reset_index()
add1_new = add1.groupby('user_id')['times_p_order'].sum().to_frame('total_products')
 add1= pd.merge(add1,add1_new,on=["user_id"],how="left")
 add1["percentage_of_bought_products_in_total"] = add1.times_p_order/add1.total_products
del add1_new
add1 = add1.drop(["times_p_order","total_products"],axis=1)
 add1.head()
%%writefile create_folds.py
if __name__ == "__main__":
df = pd.read_csv(".. /input/train.csv")
 print(df.head())
 df.loc[:, 'kfold'] = -1
df = df.sample(frac=1 ).reset_index(drop=True)
 X = df.image_id.values
y = df[['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']].values
mskf = MultilabelStratifiedKFold(n_splits=5)
 for fold,(trn_, val_)in enumerate(mskf.split(X, y)) :
print("TRAIN: ", trn_, "VAL: ", val_)
 df.loc[val_, "kfold"] = fold
print(df.kfold.value_counts())
 df.to_csv(".. /input/train_folds.csv", index=False )<load_pretrained>
summ_priordays_per_product = op.groupby(['product_id' ])['days_since_prior_order'].sum().to_frame('sum_prior_days_per_product')
 summ_priordays_per_product.head()
%%writefile create_image_pickles.py
if __name__ == "__main__":
files = glob.glob(".. /input/train_*.parquet")
 for f in files:
df = pd.read_parquet(f, engine='fastparquet')
 image_ids = df.image_id.values
df = df.drop("image_id", axis=1)
 image_array = df.values
for j, image_id in tqdm(enumerate(image_ids), total=len(image_ids)) :
joblib.dump(image_array[j, :], f".. /input/image_pickles/{image_id}.pkl")
<load_from_csv>
total_orders_per_prod=op.groupby(['product_id' ])['order_id'].count().to_frame('TotalOrdersPerProd')
 total_orders_per_prod.head()
%%writefile dataset.py
class BengaliDatasetTrain:
def __init__(self, folds, img_height, img_width, mean, std):
df = pd.read_csv(".. /input/train_folds.csv")
 df = df[["image_id", "grapheme_root", "vowel_diacritic", "consonant_diacritic", "kfold"]]
df = df[df.kfold.isin(folds)].reset_index(drop=True)
 self.image_ids = df.image_id.values
self.grapheme_root = df.grapheme_root.values
self.vowel_diacritic = df.vowel_diacritic.values
self.consonant_diacritic = df.consonant_diacritic.values
if len(folds)== 1:
self.aug = albumentations.Compose([
albumentations.Resize(img_height, img_width, always_apply=True),
albumentations.Normalize(mean, std, always_apply=True)
 ])
 else:
self.aug = albumentations.Compose([
albumentations.Resize(img_height, img_width, always_apply=True),
albumentations.Normalize(mean, std, always_apply=True)
 ])
 def __len__(self):
return len(self.image_ids)
 def __getitem__(self, item):
image = joblib.load(f".. /input/image_pickles/{self.image_ids[item]}.pkl")
 image = image.reshape(137, 236 ).astype(float)
 image = Image.fromarray(image ).convert("RGB")
 image = self.aug(image=np.array(image)) ["image"]
image = np.transpose(image,(2, 0, 1)).astype(np.float32)
 return {
"image": torch.tensor(image, dtype=torch.float),
"grapheme_root": torch.tensor(self.grapheme_root[item], dtype=torch.long),
"vowel_diacritic": torch.tensor(self.vowel_diacritic[item], dtype=torch.long),
"consonant_diacritic": torch.tensor(self.consonant_diacritic[item], dtype=torch.long)
 } 
<set_options>
test1=summ_priordays_per_product['sum_prior_days_per_product'].astype(str ).astype(float)
 test1.dtypes
%%writefile models.py
class ResNet34(nn.Module):
def __init__(self, pretrained):
super(ResNet34, self ).__init__()
if pretrained is True:
self.model = pretrainedmodels.__dict__["resnet34"](pretrained="imagenet")
 else:
self.model = pretrainedmodels.__dict__["resnet34"](pretrained=None)
 self.l0 = nn.Linear(512, 168)
 self.l1 = nn.Linear(512, 11)
 self.l2 = nn.Linear(512, 7)
 def forward(self, x):
bs, _, _, _ = x.shape
x = self.model.features(x)
 x = F.adaptive_avg_pool2d(x, 1 ).reshape(bs, -1)
 l0 = self.l0(x)
 l1 = self.l1(x)
 l2 = self.l2(x)
 return l0, l1, l2 
<define_variables>
test2=total_orders_per_prod['TotalOrdersPerProd'].astype(str ).astype(float)
 test2.dtypes
%%writefile train.py
DEVICE = "cuda"
TRAINING_FOLDS_CSV = os.environ.get("TRAINING_FOLDS_CSV")
 IMG_HEIGHT = int(os.environ.get("IMG_HEIGHT"))
IMG_WIDTH = int(os.environ.get("IMG_WIDTH"))
EPOCHS = int(os.environ.get("EPOCHS"))
TRAIN_BATCH_SIZE = int(os.environ.get("TRAIN_BATCH_SIZE"))
TEST_BATCH_SIZE = int(os.environ.get("TEST_BATCH_SIZE"))
MODEL_MEAN = ast.literal_eval(os.environ.get("MODEL_MEAN"))
MODEL_STD = ast.literal_eval(os.environ.get("MODEL_STD"))
TRAINING_FOLDS = ast.literal_eval(os.environ.get("TRAINING_FOLDS"))
VALIDATION_FOLDS = ast.literal_eval(os.environ.get("VALIDATION_FOLDS"))
BASE_MODEL = os.environ.get("BASE_MODEL")
 def macro_recall(pred_y, y, n_grapheme=168, n_vowel=11, n_consonant=7):
pred_y = torch.split(pred_y, [n_grapheme, n_vowel, n_consonant], dim=1)
 pred_labels = [torch.argmax(py, dim=1 ).cpu().numpy() for py in pred_y]
y = y.cpu().numpy()
recall_grapheme = sklearn.metrics.recall_score(pred_labels[0], y[:, 0], average='macro')
 recall_vowel = sklearn.metrics.recall_score(pred_labels[1], y[:, 1], average='macro')
 recall_consonant = sklearn.metrics.recall_score(pred_labels[2], y[:, 2], average='macro')
 scores = [recall_grapheme, recall_vowel, recall_consonant]
final_score = np.average(scores, weights=[2, 1, 1])
 print(f'recall: grapheme {recall_grapheme}, vowel {recall_vowel}, consonant {recall_consonant}, 'f'total {final_score}, y {y.shape}')
 return final_score
def loss_fn(outputs, targets):
o1, o2, o3 = outputs
t1, t2, t3 = targets
l1 = nn.CrossEntropyLoss()(o1, t1)
 l2 = nn.CrossEntropyLoss()(o2, t2)
 l3 = nn.CrossEntropyLoss()(o3, t3)
 return(l1 + l2 + l3)/ 3
def train(dataset, data_loader, model, optimizer):
model.train()
final_loss = 0
counter = 0
final_outputs = []
final_targets = []
for bi, d in tqdm(enumerate(data_loader), total=int(len(dataset)/data_loader.batch_size)) :
counter = counter + 1
image = d["image"]
grapheme_root = d["grapheme_root"]
vowel_diacritic = d["vowel_diacritic"]
consonant_diacritic = d["consonant_diacritic"]
image = image.to(DEVICE, dtype=torch.float)
 grapheme_root = grapheme_root.to(DEVICE, dtype=torch.long)
 vowel_diacritic = vowel_diacritic.to(DEVICE, dtype=torch.long)
 consonant_diacritic = consonant_diacritic.to(DEVICE, dtype=torch.long)
 print(image.shape)
 optimizer.zero_grad()
outputs = model(image)
 targets =(grapheme_root, vowel_diacritic, consonant_diacritic)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
final_loss += loss
o1, o2, o3 = outputs
t1, t2, t3 = targets
final_outputs.append(torch.cat(( o1,o2,o3), dim=1))
final_targets.append(torch.stack(( t1,t2,t3), dim=1))
final_outputs = torch.cat(final_outputs)
 final_targets = torch.cat(final_targets)
 print("=================Train=================")
 macro_recall_score = macro_recall(final_outputs, final_targets)
 return final_loss/counter , macro_recall_score
def evaluate(dataset, data_loader, model):
with torch.no_grad() :
model.eval()
final_loss = 0
counter = 0
final_outputs = []
final_targets = []
for bi, d in tqdm(enumerate(data_loader), total=int(len(dataset)/data_loader.batch_size)) :
counter = counter + 1
image = d["image"]
grapheme_root = d["grapheme_root"]
vowel_diacritic = d["vowel_diacritic"]
consonant_diacritic = d["consonant_diacritic"]
image = image.to(DEVICE, dtype=torch.float)
 grapheme_root = grapheme_root.to(DEVICE, dtype=torch.long)
 vowel_diacritic = vowel_diacritic.to(DEVICE, dtype=torch.long)
 consonant_diacritic = consonant_diacritic.to(DEVICE, dtype=torch.long)
 outputs = model(image)
 targets =(grapheme_root, vowel_diacritic, consonant_diacritic)
 loss = loss_fn(outputs, targets)
 final_loss += loss
o1, o2, o3 = outputs
t1, t2, t3 = targets
final_outputs.append(torch.cat(( o1,o2,o3), dim=1))
final_targets.append(torch.stack(( t1,t2,t3), dim=1))
final_outputs = torch.cat(final_outputs)
 final_targets = torch.cat(final_targets)
 print("=================Train=================")
 macro_recall_score = macro_recall(final_outputs, final_targets)
 return final_loss/counter , macro_recall_score
def main() :
model = MODEL_DISPATCHER[BASE_MODEL](pretrained=True)
 model.to(DEVICE)
 train_dataset = BengaliDatasetTrain(
 folds=TRAINING_FOLDS,
img_height = IMG_HEIGHT,
img_width = IMG_WIDTH,
mean = MODEL_MEAN,
std = MODEL_STD 
)
 train_loader = torch.utils.data.DataLoader(
 dataset=train_dataset,
batch_size= TRAIN_BATCH_SIZE,
shuffle=True,
num_workers=4 
)
 valid_dataset = BengaliDatasetTrain(
 folds=VALIDATION_FOLDS,
img_height = IMG_HEIGHT,
img_width = IMG_WIDTH,
mean = MODEL_MEAN,
std = MODEL_STD 
)
 valid_loader = torch.utils.data.DataLoader(
 dataset=valid_dataset,
batch_size= TEST_BATCH_SIZE,
shuffle=True,
num_workers=4 
)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-4)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
mode="min",
patience=5,
factor=0.3,verbose=True)
 early_stopping = EarlyStopping(patience=5, verbose=True)
 best_score = -1
print("FOLD : ", VALIDATION_FOLDS[0])
 for epoch in range(1, EPOCHS+1):
train_loss, train_score = train(train_dataset,train_loader, model, optimizer)
 val_loss, val_score = evaluate(valid_dataset, valid_loader, model)
 scheduler.step(val_loss)
 if val_score > best_score:
best_score = val_score
torch.save(model.state_dict() , f"{BASE_MODEL}_fold{VALIDATION_FOLDS[0]}.pth")
 epoch_len = len(str(EPOCHS))
print_msg =(f'[{epoch:>{epoch_len}}/{EPOCHS:>{epoch_len}}] ' +
f'train_loss: {train_loss:.5f} ' +
f'train_score: {train_score:.5f} ' +
f'valid_loss: {val_loss:.5f} ' +
f'valid_score: {val_score:.5f}' 
)
 print(print_msg)
 early_stopping(val_score, model)
 if early_stopping.early_stop:
print("Early stopping")
 break
if __name__ == "__main__":
main()<set_options>
our_ratio=our_ratio['our_ratio'].astype(str ).astype(float)
 our_ratio.dtypes
%%writefile run.sh
export IMG_HEIGHT=137
export IMG_WIDTH=236
export EPOCHS=50
export TRAIN_BATCH_SIZE=64
export TEST_BATCH_SIZE=64
export MODEL_MEAN="(0.485, 0.456, 0.406)"
export MODEL_STD="(0.229, 0.224, 0.225)"
export BASE_MODEL="resnet34"
export TRAINING_FOLDS_CSV=".. /input/train_folds.csv"
export TRAINING_FOLDS="(0,1,2,3)"
export VALIDATION_FOLDS="(4,)"
python3 train.py
export TRAINING_FOLDS="(0,1,2,4)"
export VALIDATION_FOLDS="(3,)"
python3 train.py
export TRAINING_FOLDS="(0,1,3,4)"
export VALIDATION_FOLDS="(2,)"
python3 train.py
export TRAINING_FOLDS="(0,2,3,4)"
export VALIDATION_FOLDS="(1,)"
python3 train.py
export TRAINING_FOLDS="(1,2,3,4)"
export VALIDATION_FOLDS="(0,)"
python3 train.py 
<define_variables>
user = user.merge(u_reorder, on='user_id', how='left')
 del u_reorder
gc.collect()
user.head()
pt_models = ".. /input/pretrained-models/pretrained-models.pytorch-master/"
sys.path.insert(0, pt_models)
<install_modules>
items = pd.merge(left =pd.merge(left=products, right=departments, how='left'), right=aisles, how='left')
 items.head()
<import_modules>
prd = op.groupby('product_id')['order_id'].count().to_frame('p_total_purchases')
 prd= prd.reset_index()
import glob
import torch
import albumentations
import pandas as pd
import numpy as np
from tqdm import tqdm
from PIL import Image
import joblib
import torch.nn as nn
from torch.nn import functional as F<define_search_space>
items = pd.merge(left =pd.merge(left=products, right=departments, how='left'), right=aisles, how='left')
 items.head()
MODEL_MEAN =(0.485, 0.456, 0.406)
 MODEL_STD =(0.229, 0.224, 0.225)
 IMG_HEIGHT = 137
IMG_WIDTH = 236
DEVICE="cuda"<choose_model_class>
order_products_all = pd.concat([order_products_train, order_products_prior], axis=0)
 order_products_all.head()
class ResNet34(nn.Module):
def __init__(self, pretrained):
super(ResNet34, self ).__init__()
if pretrained is True:
self.model = pretrainedmodels.__dict__["resnet34"](pretrained="imagenet")
 else:
self.model = pretrainedmodels.__dict__["resnet34"](pretrained=None)
 self.l0 = nn.Linear(512, 168)
 self.l1 = nn.Linear(512, 11)
 self.l2 = nn.Linear(512, 7)
 def forward(self, x):
bs, _, _, _ = x.shape
x = self.model.features(x)
 x = F.adaptive_avg_pool2d(x, 1 ).reshape(bs, -1)
 l0 = self.l0(x)
 l1 = self.l1(x)
 l2 = self.l2(x)
 return l0, l1, l2 
<normalization>
grouped = order_products_all.groupby("reordered")["product_id"].aggregate({'Total_products': 'count'} ).reset_index()
grouped['Ratios'] = grouped["Total_products"].apply(lambda x: x /grouped['Total_products'].sum())
 grouped.head()
class BengaliDatasetTest:
def __init__(self, df, img_height, img_width, mean, std):
self.image_ids = df.image_id.values
self.img_arr = df.iloc[:, 1:].values
self.aug = albumentations.Compose([
albumentations.Resize(img_height, img_width, always_apply=True),
albumentations.Normalize(mean, std, always_apply=True)
 ])
 def __len__(self):
return len(self.image_ids)
 def __getitem__(self, item):
image = self.img_arr[item, :]
img_id = self.image_ids[item]
image = image.reshape(137, 236 ).astype(float)
 image = Image.fromarray(image ).convert("RGB")
 image = self.aug(image=np.array(image)) ["image"]
image = np.transpose(image,(2, 0, 1)).astype(np.float32)
 return {
"image": torch.tensor(image, dtype=torch.float),
"image_id": img_id
}<create_dataframe>
grouped1 = items.groupby("department")["product_id"].aggregate({'Total_products': 'count'} ).reset_index()
grouped1['RatioDepartments'] = grouped1["Total_products"].apply(lambda x: x /grouped['Total_products'].sum())
 grouped1.head()
def model_predict() :
g_pred, v_pred, c_pred = [], [], []
img_ids_list = []
for file_idx in range(4):
df = pd.read_parquet(f".. /input/bengaliai-cv19/test_image_data_{file_idx}.parquet")
 dataset = BengaliDatasetTest(df=df,
img_height=IMG_HEIGHT,
img_width=IMG_WIDTH,
mean=MODEL_MEAN,
std=MODEL_STD)
 data_loader = torch.utils.data.DataLoader(
 dataset=dataset,
batch_size= TEST_BATCH_SIZE,
shuffle=False,
num_workers=4 
)
 for bi, d in enumerate(data_loader):
image = d["image"]
img_id = d["image_id"]
image = image.to(DEVICE, dtype=torch.float)
 g, v, c = model(image)
 for ii, imid in enumerate(img_id):
g_pred.append(g[ii].cpu().detach().numpy())
 v_pred.append(v[ii].cpu().detach().numpy())
 c_pred.append(c[ii].cpu().detach().numpy())
 img_ids_list.append(imid)
 return g_pred, v_pred, c_pred, img_ids_list<choose_model_class>
grouped2 = items.groupby("aisle")["product_id"].aggregate({'Total_products': 'count'} ).reset_index()
grouped2['RatioAisles'] = grouped2["Total_products"].apply(lambda x: x /grouped['Total_products'].sum())
 grouped2.head()
model = ResNet34(pretrained=False)
 TEST_BATCH_SIZE = 32
final_g_pred = []
final_v_pred = []
final_c_pred = []
final_img_ids = []
for i in range(5):
model.load_state_dict(torch.load(f".. /input/resnet34weights/resnet34_fold{i}.pth"))
model.to(DEVICE)
 model.eval()
g_pred, v_pred, c_pred, img_ids_list = model_predict()
final_g_pred.append(g_pred)
 final_v_pred.append(v_pred)
 final_c_pred.append(c_pred)
 if i == 0:
final_img_ids.extend(img_ids_list )<prepare_output>
p_reorder = op.groupby('product_id' ).filter(lambda x: x.shape[0] >40)
 p_reorder.head()
final_g = np.argmax(np.mean(np.array(final_g_pred), axis=0), axis=1)
 final_v = np.argmax(np.mean(np.array(final_v_pred), axis=0), axis=1)
 final_c = np.argmax(np.mean(np.array(final_c_pred), axis=0), axis=1 )<define_variables>
prd = prd.merge(our_ratio, on='product_id', how='left')
 prd = prd.merge(add1,on= 'product_id', how='left')
 prd.head()
predictions = []
for ii, imid in enumerate(final_img_ids):
predictions.append(( f"{imid}_grapheme_root", final_g[ii]))
predictions.append(( f"{imid}_vowel_diacritic", final_v[ii]))
predictions.append(( f"{imid}_consonant_diacritic", final_c[ii]))<create_dataframe>
prd = prd.merge(p_reorder, on='product_id', how='left')
 del p_reorder
gc.collect()
prd.head()
sub = pd.DataFrame(predictions, columns=["row_id", "target"] )<save_to_csv>
prd['p_reorder_ratio'] = prd['p_reorder_ratio'].fillna(value=0)
 prd.head()
sub.to_csv("submission.csv", index=False )<import_modules>
uxp = op.groupby(['user_id', 'product_id'])['order_id'].count().to_frame('uxp_total_bought')
 uxp.head()
import pretrainedmodels<import_modules>
times = op.groupby(['user_id', 'product_id'])[['order_id']].count()
times.columns = ['Times_Bought_N']
times.head()
import os
import numpy as np
import pandas as pd
import ast
import torch
from tqdm import tqdm
import torch.nn as nn
import joblib
import albumentations
from PIL import Image
import glob
from torch.nn import functional as F<define_search_space>
total_orders = op.groupby('user_id')['order_number'].max().to_frame('total_orders')
 total_orders = total_orders.reset_index()
total_orders.head()
IMG_HEIGHT = 137
IMG_WIDTH = 236
TEST_BATCH_SIZE = 16
MODEL_MEAN =(0.485, 0.456, 0.406)
 MODEL_STD =(0.229, 0.224, 0.225)
 DEVICE = "cuda"<feature_engineering>
span = pd.merge(total_orders, first_order_no, on='user_id', how='right')
 span.head()
class Resnet34(nn.Module):
def __init__(self, pretrained):
super(Resnet34, self ).__init__()
if pretrained is True:
self.model = pretrainedmodels.__dict__['resnet34'](pretrained = "imagenet")
 else:
self.model = pretrainedmodels.__dict__['resnet34'](pretrained = None)
 self.l0 = nn.Linear(512,168)
 self.l1 = nn.Linear(512,11)
 self.l2 = nn.Linear(512,7)
 def forward(self, x):
bs, _,_,_ = x.shape
x = self.model.features(x)
 x = F.adaptive_avg_pool2d(x,1 ).reshape(bs, -1)
 l0 = self.l0(x)
 l1 = self.l1(x)
 l2 = self.l2(x)
 return l0, l1, l2<normalization>
span['Order_Range_D'] = span.total_orders - span.first_order_number + 1
span.head()
class BengaliDatasetTrain:
def __init__(self, df, img_height, img_width, mean, std):
self.image_ids = df.image_id.values
self.img_arr = df.iloc[:, 1:].values
self.aug = albumentations.Compose([
albumentations.Resize(img_height, img_width, always_apply=True),
albumentations.Normalize(mean, std, always_apply = True)
 ])
 def __len__(self):
return len(self.image_ids)
 def __getitem__(self, item):
image = self.img_arr[item, :]
img_id = self.image_ids[item]
image = image.reshape(137,236 ).astype(float)
 image = Image.fromarray(image ).convert("RGB")
 image = self.aug(image = np.array(image)) ['image']
image = np.transpose(image,(2,0,1)).astype(np.float32)
 return{
'image': torch.tensor(image, dtype=torch.float),
"image_id":img_id
} 
<load_pretrained>
uxp_ratio = pd.merge(times, span, on=['user_id', 'product_id'], how='left')
 uxp_ratio.head()
model = Resnet34(pretrained=False)
 model.load_state_dict(torch.load(".. /input/bengali-train-model/resnet34_4_weighted.pth", map_location=torch.device('cpu')))
 model.eval()<load_from_csv>
uxp_ratio['uxp_reorder_ratio'] = uxp_ratio.Times_Bought_N / uxp_ratio.Order_Range_D
uxp_ratio.head()
predictions = []
for file_idx in range(4):
df = pd.read_parquet(f".. /input/bengaliai-cv19/test_image_data_{file_idx}.parquet")
 dataset = BengaliDatasetTrain(df = df,
img_height=IMG_HEIGHT,
img_width=IMG_WIDTH,
mean = MODEL_MEAN,
std=MODEL_STD)
 data_loader = torch.utils.data.DataLoader(dataset,
batch_size=TEST_BATCH_SIZE,
shuffle=False,
num_workers=4)
 with torch.no_grad() :
for bi, d in enumerate(data_loader):
image = d['image']
img_id = d['image_id']
g, v, c = model(image)
 g = np.argmax(g, axis=1)
 v = np.argmax(v, axis=1)
 c = np.argmax(c, axis=1)
 for ii, imid in enumerate(img_id):
predictions.append(( f"{imid}_grapheme_root", int(g[ii])))
 predictions.append(( f"{imid}_vowel_diacritic", int(v[ii])))
 predictions.append(( f"{imid}_consonant_diacritic", int(c[ii])) )<create_dataframe>
uxp_ratio = uxp_ratio.drop(['Times_Bought_N', 'total_orders', 'first_order_number', 'Order_Range_D'], axis=1)
 uxp_ratio.head()
sub = pd.DataFrame(predictions, columns = ['row_id', 'target'] )<save_to_csv>
del [times, first_order_no, span]
sub.to_csv('submission.csv', index=False )<save_to_csv>
uxp = uxp.merge(uxp_ratio, on=['user_id', 'product_id'], how='left')
 del uxp_ratio
uxp.head()
sub.to_csv('submission.csv', index=False )<load_from_csv>
data = uxp.merge(user, on='user_id', how='left')
 data.head()
%%writefile create_folds.py
if __name__ == "__main__":
df = pd.read_csv(".. /input/train.csv")
 print(df.head())
 df.loc[:, 'kfold'] = -1
df = df.sample(frac=1 ).reset_index(drop=True)
 X = df.image_id.values
y = df[['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']].values
mskf = MultilabelStratifiedKFold(n_splits=5)
 for fold,(trn_, val_)in enumerate(mskf.split(X, y)) :
print("TRAIN: ", trn_, "VAL: ", val_)
 df.loc[val_, "kfold"] = fold
print(df.kfold.value_counts())
 df.to_csv(".. /input/train_folds.csv", index=False )<load_pretrained>
data = data.merge(prd, on='product_id', how='left')
 data = data.merge(add1, on=["user_id","product_id"], how="left")
 data.head()
%%writefile create_image_pickles.py
if __name__ == "__main__":
files = glob.glob(".. /input/train_*.parquet")
 for f in files:
df = pd.read_parquet(f, engine='fastparquet')
 image_ids = df.image_id.values
df = df.drop("image_id", axis=1)
 image_array = df.values
for j, image_id in tqdm(enumerate(image_ids), total=len(image_ids)) :
joblib.dump(image_array[j, :], f".. /input/image_pickles/{image_id}.pkl")
<load_from_csv>
del op, user, prd, uxp
gc.collect()
%%writefile dataset.py
class BengaliDatasetTrain:
def __init__(self, folds, img_height, img_width, mean, std):
df = pd.read_csv(".. /input/train_folds.csv")
 df = df[["image_id", "grapheme_root", "vowel_diacritic", "consonant_diacritic", "kfold"]]
df = df[df.kfold.isin(folds)].reset_index(drop=True)
 self.image_ids = df.image_id.values
self.grapheme_root = df.grapheme_root.values
self.vowel_diacritic = df.vowel_diacritic.values
self.consonant_diacritic = df.consonant_diacritic.values
if len(folds)== 1:
self.aug = albumentations.Compose([
albumentations.Resize(img_height, img_width, always_apply=True),
albumentations.Normalize(mean, std, always_apply=True)
 ])
 else:
self.aug = albumentations.Compose([
albumentations.Resize(img_height, img_width, always_apply=True),
albumentations.Normalize(mean, std, always_apply=True)
 ])
 def __len__(self):
return len(self.image_ids)
 def __getitem__(self, item):
image = joblib.load(f".. /input/image_pickles/{self.image_ids[item]}.pkl")
 image = image.reshape(137, 236 ).astype(float)
 image = Image.fromarray(image ).convert("RGB")
 image = self.aug(image=np.array(image)) ["image"]
image = np.transpose(image,(2, 0, 1)).astype(np.float32)
 return {
"image": torch.tensor(image, dtype=torch.float),
"grapheme_root": torch.tensor(self.grapheme_root[item], dtype=torch.long),
"vowel_diacritic": torch.tensor(self.vowel_diacritic[item], dtype=torch.long),
"consonant_diacritic": torch.tensor(self.consonant_diacritic[item], dtype=torch.long)
 } 
<set_options>
orders_future = orders[(( orders.eval_set=='train')|(orders.eval_set=='test')) ]
orders_future = orders_future[ ['user_id', 'eval_set', 'order_id'] ]
orders_future.head(10)
%%writefile models.py
class ResNet34(nn.Module):
def __init__(self, pretrained):
super(ResNet34, self ).__init__()
if pretrained is True:
self.model = pretrainedmodels.__dict__["resnet34"](pretrained="imagenet")
 else:
self.model = pretrainedmodels.__dict__["resnet34"](pretrained=None)
 self.l0 = nn.Linear(512, 168)
 self.l1 = nn.Linear(512, 11)
 self.l2 = nn.Linear(512, 7)
 def forward(self, x):
bs, _, _, _ = x.shape
x = self.model.features(x)
 x = F.adaptive_avg_pool2d(x, 1 ).reshape(bs, -1)
 l0 = self.l0(x)
 l1 = self.l1(x)
 l2 = self.l2(x)
 return l0, l1, l2 
<define_variables>
data = data.merge(orders_future, on='user_id', how='left')
 data.head(10 )
%%writefile train.py
DEVICE = "cuda"
TRAINING_FOLDS_CSV = os.environ.get("TRAINING_FOLDS_CSV")
 IMG_HEIGHT = int(os.environ.get("IMG_HEIGHT"))
IMG_WIDTH = int(os.environ.get("IMG_WIDTH"))
EPOCHS = int(os.environ.get("EPOCHS"))
TRAIN_BATCH_SIZE = int(os.environ.get("TRAIN_BATCH_SIZE"))
TEST_BATCH_SIZE = int(os.environ.get("TEST_BATCH_SIZE"))
MODEL_MEAN = ast.literal_eval(os.environ.get("MODEL_MEAN"))
MODEL_STD = ast.literal_eval(os.environ.get("MODEL_STD"))
TRAINING_FOLDS = ast.literal_eval(os.environ.get("TRAINING_FOLDS"))
VALIDATION_FOLDS = ast.literal_eval(os.environ.get("VALIDATION_FOLDS"))
BASE_MODEL = os.environ.get("BASE_MODEL")
 def macro_recall(pred_y, y, n_grapheme=168, n_vowel=11, n_consonant=7):
pred_y = torch.split(pred_y, [n_grapheme, n_vowel, n_consonant], dim=1)
 pred_labels = [torch.argmax(py, dim=1 ).cpu().numpy() for py in pred_y]
y = y.cpu().numpy()
recall_grapheme = sklearn.metrics.recall_score(pred_labels[0], y[:, 0], average='macro')
 recall_vowel = sklearn.metrics.recall_score(pred_labels[1], y[:, 1], average='macro')
 recall_consonant = sklearn.metrics.recall_score(pred_labels[2], y[:, 2], average='macro')
 scores = [recall_grapheme, recall_vowel, recall_consonant]
final_score = np.average(scores, weights=[2, 1, 1])
 print(f'recall: grapheme {recall_grapheme}, vowel {recall_vowel}, consonant {recall_consonant}, 'f'total {final_score}, y {y.shape}')
 return final_score
def loss_fn(outputs, targets):
o1, o2, o3 = outputs
t1, t2, t3 = targets
l1 = nn.CrossEntropyLoss()(o1, t1)
 l2 = nn.CrossEntropyLoss()(o2, t2)
 l3 = nn.CrossEntropyLoss()(o3, t3)
 return(l1 + l2 + l3)/ 3
def train(dataset, data_loader, model, optimizer):
model.train()
final_loss = 0
counter = 0
final_outputs = []
final_targets = []
for bi, d in tqdm(enumerate(data_loader), total=int(len(dataset)/data_loader.batch_size)) :
counter = counter + 1
image = d["image"]
grapheme_root = d["grapheme_root"]
vowel_diacritic = d["vowel_diacritic"]
consonant_diacritic = d["consonant_diacritic"]
image = image.to(DEVICE, dtype=torch.float)
 grapheme_root = grapheme_root.to(DEVICE, dtype=torch.long)
 vowel_diacritic = vowel_diacritic.to(DEVICE, dtype=torch.long)
 consonant_diacritic = consonant_diacritic.to(DEVICE, dtype=torch.long)
 print(image.shape)
 optimizer.zero_grad()
outputs = model(image)
 targets =(grapheme_root, vowel_diacritic, consonant_diacritic)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
final_loss += loss
o1, o2, o3 = outputs
t1, t2, t3 = targets
final_outputs.append(torch.cat(( o1,o2,o3), dim=1))
final_targets.append(torch.stack(( t1,t2,t3), dim=1))
final_outputs = torch.cat(final_outputs)
 final_targets = torch.cat(final_targets)
 print("=================Train=================")
 macro_recall_score = macro_recall(final_outputs, final_targets)
 return final_loss/counter , macro_recall_score
def evaluate(dataset, data_loader, model):
with torch.no_grad() :
model.eval()
final_loss = 0
counter = 0
final_outputs = []
final_targets = []
for bi, d in tqdm(enumerate(data_loader), total=int(len(dataset)/data_loader.batch_size)) :
counter = counter + 1
image = d["image"]
grapheme_root = d["grapheme_root"]
vowel_diacritic = d["vowel_diacritic"]
consonant_diacritic = d["consonant_diacritic"]
image = image.to(DEVICE, dtype=torch.float)
 grapheme_root = grapheme_root.to(DEVICE, dtype=torch.long)
 vowel_diacritic = vowel_diacritic.to(DEVICE, dtype=torch.long)
 consonant_diacritic = consonant_diacritic.to(DEVICE, dtype=torch.long)
 outputs = model(image)
 targets =(grapheme_root, vowel_diacritic, consonant_diacritic)
 loss = loss_fn(outputs, targets)
 final_loss += loss
o1, o2, o3 = outputs
t1, t2, t3 = targets
final_outputs.append(torch.cat(( o1,o2,o3), dim=1))
final_targets.append(torch.stack(( t1,t2,t3), dim=1))
final_outputs = torch.cat(final_outputs)
 final_targets = torch.cat(final_targets)
 print("=================Train=================")
 macro_recall_score = macro_recall(final_outputs, final_targets)
 return final_loss/counter , macro_recall_score
def main() :
model = MODEL_DISPATCHER[BASE_MODEL](pretrained=True)
 model.to(DEVICE)
 train_dataset = BengaliDatasetTrain(
 folds=TRAINING_FOLDS,
img_height = IMG_HEIGHT,
img_width = IMG_WIDTH,
mean = MODEL_MEAN,
std = MODEL_STD 
)
 train_loader = torch.utils.data.DataLoader(
 dataset=train_dataset,
batch_size= TRAIN_BATCH_SIZE,
shuffle=True,
num_workers=4 
)
 valid_dataset = BengaliDatasetTrain(
 folds=VALIDATION_FOLDS,
img_height = IMG_HEIGHT,
img_width = IMG_WIDTH,
mean = MODEL_MEAN,
std = MODEL_STD 
)
 valid_loader = torch.utils.data.DataLoader(
 dataset=valid_dataset,
batch_size= TEST_BATCH_SIZE,
shuffle=True,
num_workers=4 
)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-4)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
mode="min",
patience=5,
factor=0.3,verbose=True)
 early_stopping = EarlyStopping(patience=5, verbose=True)
 best_score = -1
print("FOLD : ", VALIDATION_FOLDS[0])
 for epoch in range(1, EPOCHS+1):
train_loss, train_score = train(train_dataset,train_loader, model, optimizer)
 val_loss, val_score = evaluate(valid_dataset, valid_loader, model)
 scheduler.step(val_loss)
 if val_score > best_score:
best_score = val_score
torch.save(model.state_dict() , f"{BASE_MODEL}_fold{VALIDATION_FOLDS[0]}.pth")
 epoch_len = len(str(EPOCHS))
print_msg =(f'[{epoch:>{epoch_len}}/{EPOCHS:>{epoch_len}}] ' +
f'train_loss: {train_loss:.5f} ' +
f'train_score: {train_score:.5f} ' +
f'valid_loss: {val_loss:.5f} ' +
f'valid_score: {val_score:.5f}' 
)
 print(print_msg)
 early_stopping(val_score, model)
 if early_stopping.early_stop:
print("Early stopping")
 break
if __name__ == "__main__":
main()<set_options>
data_train = data[data.eval_set=='train']
data_train.head()
%%writefile run.sh
export IMG_HEIGHT=137
export IMG_WIDTH=236
export EPOCHS=50
export TRAIN_BATCH_SIZE=64
export TEST_BATCH_SIZE=64
export MODEL_MEAN="(0.485, 0.456, 0.406)"
export MODEL_STD="(0.229, 0.224, 0.225)"
export BASE_MODEL="resnet34"
export TRAINING_FOLDS_CSV=".. /input/train_folds.csv"
export TRAINING_FOLDS="(0,1,2,3)"
export VALIDATION_FOLDS="(4,)"
python3 train.py
export TRAINING_FOLDS="(0,1,2,4)"
export VALIDATION_FOLDS="(3,)"
python3 train.py
export TRAINING_FOLDS="(0,1,3,4)"
export VALIDATION_FOLDS="(2,)"
python3 train.py
export TRAINING_FOLDS="(0,2,3,4)"
export VALIDATION_FOLDS="(1,)"
python3 train.py
export TRAINING_FOLDS="(1,2,3,4)"
export VALIDATION_FOLDS="(0,)"
python3 train.py 
<define_variables>
data_train = data_train.merge(order_products_train[['product_id','order_id', 'reordered']], on=['product_id','order_id'], how='left')
 data_train.head(15 )
pt_models = ".. /input/pretrained-models/pretrained-models.pytorch-master/"
sys.path.insert(0, pt_models)
<import_modules>
data_train['reordered'] = data_train['reordered'].fillna(0)
 data_train.head(15 )
import glob
import torch
import albumentations
import pandas as pd
import numpy as np
from tqdm import tqdm
from PIL import Image
import joblib
import torch.nn as nn
from torch.nn import functional as F<define_search_space>
data_train = data_train.set_index(['user_id', 'product_id'])
 data_train.head(15 )
MODEL_MEAN =(0.485, 0.456, 0.406)
 MODEL_STD =(0.229, 0.224, 0.225)
 IMG_HEIGHT = 137
IMG_WIDTH = 236
DEVICE="cuda"<choose_model_class>
data_train = data_train.drop(['eval_set', 'order_id'], axis=1)
 data_train.head(15 )
class ResNet34(nn.Module):
def __init__(self, pretrained):
super(ResNet34, self ).__init__()
if pretrained is True:
self.model = pretrainedmodels.__dict__["resnet34"](pretrained="imagenet")
 else:
self.model = pretrainedmodels.__dict__["resnet34"](pretrained=None)
 self.l0 = nn.Linear(512, 168)
 self.l1 = nn.Linear(512, 11)
 self.l2 = nn.Linear(512, 7)
 def forward(self, x):
bs, _, _, _ = x.shape
x = self.model.features(x)
 x = F.adaptive_avg_pool2d(x, 1 ).reshape(bs, -1)
 l0 = self.l0(x)
 l1 = self.l1(x)
 l2 = self.l2(x)
 return l0, l1, l2 
<normalization>
data_test = data[data.eval_set=='test']
data_test.head()
class BengaliDatasetTest:
def __init__(self, df, img_height, img_width, mean, std):
self.image_ids = df.image_id.values
self.img_arr = df.iloc[:, 1:].values
self.aug = albumentations.Compose([
albumentations.Resize(img_height, img_width, always_apply=True),
albumentations.Normalize(mean, std, always_apply=True)
 ])
 def __len__(self):
return len(self.image_ids)
 def __getitem__(self, item):
image = self.img_arr[item, :]
img_id = self.image_ids[item]
image = image.reshape(137, 236 ).astype(float)
 image = Image.fromarray(image ).convert("RGB")
 image = self.aug(image=np.array(image)) ["image"]
image = np.transpose(image,(2, 0, 1)).astype(np.float32)
 return {
"image": torch.tensor(image, dtype=torch.float),
"image_id": img_id
}<create_dataframe>
data_test = data_test.set_index(['user_id', 'product_id'])
 data_test.head()
def model_predict() :
g_pred, v_pred, c_pred = [], [], []
img_ids_list = []
for file_idx in range(4):
df = pd.read_parquet(f".. /input/bengaliai-cv19/test_image_data_{file_idx}.parquet")
 dataset = BengaliDatasetTest(df=df,
img_height=IMG_HEIGHT,
img_width=IMG_WIDTH,
mean=MODEL_MEAN,
std=MODEL_STD)
 data_loader = torch.utils.data.DataLoader(
 dataset=dataset,
batch_size= TEST_BATCH_SIZE,
shuffle=False,
num_workers=4 
)
 for bi, d in enumerate(data_loader):
image = d["image"]
img_id = d["image_id"]
image = image.to(DEVICE, dtype=torch.float)
 g, v, c = model(image)
 for ii, imid in enumerate(img_id):
g_pred.append(g[ii].cpu().detach().numpy())
 v_pred.append(v[ii].cpu().detach().numpy())
 c_pred.append(c[ii].cpu().detach().numpy())
 img_ids_list.append(imid)
 return g_pred, v_pred, c_pred, img_ids_list<choose_model_class>
data_test = data_test.drop(['eval_set','order_id'], axis=1)
 data_test.head()
model = ResNet34(pretrained=False)
 TEST_BATCH_SIZE = 32
final_g_pred = []
final_v_pred = []
final_c_pred = []
final_img_ids = []
for i in range(5):
model.load_state_dict(torch.load(f".. /input/resnet34weights/resnet34_fold{i}.pth"))
model.to(DEVICE)
 model.eval()
g_pred, v_pred, c_pred, img_ids_list = model_predict()
final_g_pred.append(g_pred)
 final_v_pred.append(v_pred)
 final_c_pred.append(c_pred)
 if i == 0:
final_img_ids.extend(img_ids_list )<prepare_output>
final_g = np.argmax(np.mean(np.array(final_g_pred), axis=0), axis=1)
 final_v = np.argmax(np.mean(np.array(final_v_pred), axis=0), axis=1)
 final_c = np.argmax(np.mean(np.array(final_c_pred), axis=0), axis=1 )<define_variables>
X_train, X_val, y_train, y_val = train_test_split(data_train.drop('reordered', axis=1), data_train.reordered, test_size=0.8, random_state=42)
 rfc = RandomForestClassifier(n_estimators=12, n_jobs=-1 ,random_state=42)
 model = rfc.fit(X_train, y_train)
 y_val_pred =(model.predict_proba(X_val)[:,1] >= 0.22 ).astype(int)
 print("RESULTS ON VALIDATION SET
====================")
 print("F1 Score: ",f1_score(y_val, y_val_pred, average='binary'), "
====================")
 print("Classification Report
", classification_report(y_val, y_val_pred), "
====================")
 print("Confusion Matrix
", confusion_matrix(y_val, y_val_pred))
del [X_val, y_val]
predictions = []
for ii, imid in enumerate(final_img_ids):
predictions.append(( f"{imid}_grapheme_root", final_g[ii]))
predictions.append(( f"{imid}_vowel_diacritic", final_v[ii]))
predictions.append(( f"{imid}_consonant_diacritic", final_c[ii]))<create_dataframe>
test_pred = model.predict(data_test ).astype(int)
 test_pred =(model.predict_proba(data_test)[:,1] >= 0.22 ).astype(int)
 test_pred[0:20]
sub = pd.DataFrame(predictions, columns=["row_id", "target"] )<save_to_csv>
data_test['prediction'] = test_pred
data_test.head(10 )
sub.to_csv("submission.csv", index=False )<install_modules>
orders_test = orders.loc[orders.eval_set=='test',("user_id", "order_id")]
orders_test.head()
!pip install '.. /input/kerasefficientnetb3/efficientnet-1.0.0-py3-none-any.whl' 
<define_variables>
final = final.merge(orders_test, on='user_id', how='left')
 final.head()
HEIGHT = 137
WIDTH = 236
FACTOR = 0.70
HEIGHT_NEW = int(HEIGHT * FACTOR)
 WIDTH_NEW = int(WIDTH * FACTOR)
 CHANNELS = 3
BATCH_SIZE = 16
DIR = '.. /input/bengaliai-cv19'<train_model>
final = final.drop('user_id', axis=1)
print(HEIGHT_NEW)
 print(WIDTH_NEW)
 def resize_image(img, WIDTH_NEW, HEIGHT_NEW):
img = 255 - img
img =(img *(255.0 / img.max())).astype(np.uint8)
 img = img.reshape(HEIGHT, WIDTH)
 image_resized = cv2.resize(img,(WIDTH_NEW, HEIGHT_NEW), interpolation = cv2.INTER_AREA)
 return image_resized.reshape(-1 )<compute_train_metric>
final['product_id'] = final.product_id.astype(int)
 del orders
del orders_test
gc.collect()
final.head()
gm_exp = tf.Variable(3.0, dtype = tf.float32)
 def generalized_mean_pool_2d(X):
pool =(tf.reduce_mean(tf.abs(X**(gm_exp)) ,
axis = [1, 2],
keepdims = False)+ 1.e-7)**(1./gm_exp)
 return pool<choose_model_class>
d = dict()
for row in final.itertuples() :
if row.prediction== 1:
try:
d[row.order_id] += ' ' + str(row.product_id)
 except:
d[row.order_id] = str(row.product_id)
 for order in final.order_id:
if order not in d:
d[order] = 'None'
gc.collect()
d
def create_model(input_shape):
input = Input(shape = input_shape)
 x_model = efn.EfficientNetB3(weights = None, include_top = False, input_tensor = input, pooling = None, classes = None)
 for layer in x_model.layers:
layer.trainable = True
lambda_layer = Lambda(generalized_mean_pool_2d)
 lambda_layer.trainable_weights.extend([gm_exp])
 x = lambda_layer(x_model.output)
 grapheme_root = Dense(168, activation = 'softmax', name = 'root' )(x)
 vowel_diacritic = Dense(11, activation = 'softmax', name = 'vowel' )(x)
 consonant_diacritic = Dense(7, activation = 'softmax', name = 'consonant' )(x)
 model = Model(inputs = x_model.input, outputs = [grapheme_root, vowel_diacritic, consonant_diacritic])
 return model<choose_model_class>
sub = pd.DataFrame.from_dict(d, orient='index')
 sub.reset_index(inplace=True)
 sub.columns = ['order_id', 'products']
sub.head()
<load_pretrained><EOS> 
sub.to_csv('sub.csv', index=False )