Kaggle competition: Instacart Market Basket Analysis
<SOS> metric: MeanFScore Kaggle data source: instacart-market-basket-analysis<train_model>
gc.enable()
print('Training Data:',train.info() , end = "
")
 print('Testing Data:',test.info() )<count_unique_values>
orders = pd.read_csv('.. /input/orders.csv')
 order_products_train = pd.read_csv('.. /input/order_products__train.csv')
 order_products_prior = pd.read_csv('.. /input/order_products__prior.csv')
 products = pd.read_csv('.. /input/products.csv')
 aisles = pd.read_csv('.. /input/aisles.csv')
 departments = pd.read_csv('.. /input/departments.csv' )
print(train.shape, test.shape, sub.shape, data.shape)
 print(train.Patient.nunique() , test.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique() )<feature_engineering>
aisles['aisle'] = aisles['aisle'].astype('category')
 departments['department'] = departments['department'].astype('category')
 orders['eval_set'] = orders['eval_set'].astype('category')
 products['product_name'] = products['product_name'].astype('category' )
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )<categorify>
op = orders.merge(order_products_prior, on='order_id', how='inner')
 op.head()
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )<merge>
user = op.groupby('user_id')['order_number'].max().to_frame('u_total_orders')
 user = user.reset_index()
user.head()
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base<data_type_conversions>
user = user.merge(u_reorder, on='user_id', how='left')
 del u_reorder
gc.collect()
user.head()
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )<feature_engineering>
prd = op.groupby('product_id')['order_id'].count().to_frame('p_total_purchases')
 prd = prd.reset_index()
prd.head()
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']<filter>
p_reorder = op.groupby('product_id' ).filter(lambda x: x.shape[0] >40)
 p_reorder.head()
train = data.loc[data.WHERE=='train']
test = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data<compute_test_metric>
p_reorder = p_reorder.groupby('product_id')['reordered'].mean().to_frame('p_reorder_ratio')
 p_reorder = p_reorder.reset_index()
p_reorder.head()
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model() :
z = L.Input(( 9,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model<prepare_x_and_y>
prd = prd.merge(p_reorder, on='product_id', how='left')
 del p_reorder
gc.collect()
prd.head()
y = train['FVC'].values
z = train[FE].values
ze = sub[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))<choose_model_class>
prd['p_reorder_ratio'] = prd['p_reorder_ratio'].fillna(value=0)
 prd.head()
NFOLD = 5
kf = KFold(n_splits=NFOLD )<find_best_model_class>
uxp = op.groupby(['user_id', 'product_id'])['order_id'].count().to_frame('uxp_total_bought')
 uxp = uxp.reset_index()
uxp.head()
%%time
cnt = 0
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=800,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD<compute_test_metric>
op['order_number_back'] = op.groupby('user_id')['order_number'].transform(max)- op.order_number +1
op5 = op[op.order_number_back <= 5]
last_five = op5.groupby(['user_id','product_id'])[['order_id']].count()
last_five.columns = ['times_last5']
last_five['times_last5_ratio'] = last_five.times_last5 / 5
uxp = uxp.merge(last_five , on=['user_id', 'product_id'], how='left')
 del [last_five]
gc.collect()
uxp.head()
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )<feature_engineering>
uxp['times_last5'] = uxp['times_last5'].fillna(value=0)
 uxp['times_last5_ratio'] = uxp['times_last5_ratio'].fillna(value=0)
 uxp.head()
sub['FVC1'] = 0.996 * pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]<prepare_output>
times = op5.groupby(['user_id', 'product_id'])[['order_id']].count()
times.columns = ['Times_Bought_N']
times.head()
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()<feature_engineering>
total_orders = op5.groupby('user_id')['order_number'].max().to_frame('total_orders')
 total_orders.head()
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']<load_from_csv>
span = pd.merge(total_orders, first_order_no, on='user_id', how='right')
 span.head()
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1<save_to_csv>
span['Order_Range_D'] = span.total_orders - span.first_order_number + 1
span.head()
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )<set_options>
uxp_ratio = pd.merge(times, span, on=['user_id', 'product_id'], how='left')
 uxp_ratio.head()
tqdm.pandas()
%matplotlib inline
print(os.listdir(".. /input")) 
<set_options>
uxp_ratio['uxp_order_ratio'] = uxp_ratio.Times_Bought_N / uxp_ratio.Order_Range_D
uxp_ratio.head()
def manual_seed(seed=420, cuda=False):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 if cuda:
torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
manual_seed(cuda=True )<feature_engineering>
uxp_ratio = uxp_ratio.drop(['Times_Bought_N', 'total_orders', 'first_order_number', 'Order_Range_D'], axis=1)
 uxp_ratio.head()
@contextmanager
def timer(msg):
start = time.time()
print(f'[{msg}] start...')
 yield
elapsed = time.time() - start
hours, rem = divmod(elapsed, 3600)
 minutes, seconds = divmod(rem, 60)
 elapsed_str = "{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)
 print(f'[{msg}] done in {elapsed_str}.' )<define_variables>
del [times, first_order_no, span,op5]
MAX_VOCAB_SIZE = 100000
MAX_LEN = 256
TARGET_COLUMNS = ['target']<define_variables>
uxp = uxp.merge(uxp_ratio, on=['user_id', 'product_id'], how='left')
 del uxp_ratio
uxp.head()
identity_columns = ['asian', 'atheist',
'bisexual', 'black', 'buddhist', 'christian', 'female',
'heterosexual', 'hindu', 'homosexual_gay_or_lesbian',
'intellectual_or_learning_disability', 'jewish', 'latino', 'male',
'muslim', 'other_disability', 'other_gender',
'other_race_or_ethnicity', 'other_religion',
'other_sexual_orientation', 'physical_disability',
'psychiatric_or_mental_illness', 'transgender', 'white']<define_variables>
data = uxp.merge(user, on='user_id', how='left')
 data.head()
special_characters = {
"’": "'", "‘": "'", "´": "'", "`": "'", "…": "...", "&": " and ", "“": '"', "”": '"',
"⁰": "0", "¹": "1", "²": "2", "³": "3", "⁴": "4", "⁵": "5", "⁶": "6", "⁷": "7", "⁸": "8", "⁹": "9",
"₀": "0", "₁": "1", "₂": "2", "₃": "3", "₄": "4", "₅": "5", "₆": "6", "₇": "7", "₈": "8", "₉": "9",
"ᴀ": "a", "ʙ": "b", "ᴄ": "c", "ᴅ": "d", "ᴇ": "e", "ғ": "f", "ɢ": "g", "ʜ": "h", "ɪ": "i",
"ᴊ": "j", "ᴋ": "k", "ʟ": "l", "ᴍ": "m", "ɴ": "n", "ᴏ": "o", "ᴘ": "p", "ǫ": "q", "ʀ": "r",
"s": "s", "ᴛ": "t", "ᴜ": "u", "ᴠ": "v", "ᴡ": "w", "x": "x", "ʏ": "y", "ᴢ": "z"
}
contractions = json.load(open('.. /input/english-contractions/contractions.json', 'r'))
contractions = {key.lower() :value.lower() for key, value in contractions.items() }
punct = "/-'?!.,
special_characters_re = re.compile('({})'.format('|'.join(special_characters.keys())))
 special_characters_map = lambda match: special_characters[match.group(0)]
contractions_re = re.compile('({})'.format('|'.join(contractions.keys())))
 contractions_map = lambda match: contractions[match.group(0)]
punct_table = str.maketrans(punct, ' '*len(punct))
def tokenize(text):
text = text.lower()
text = special_characters_re.sub(special_characters_map, text)
 text = contractions_re.sub(contractions_map, text)
 text = text.translate(punct_table)
 tokens = text_to_word_sequence(text, lower=False)
 return tokens<load_from_csv>
data = data.merge(prd, on='product_id', how='left')
 data.head()
with timer('Process train.csv'):
print("Loading File...")
 train_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 print("Filling NaNs with 0s")
 train_df.fillna(0.0, inplace=True)
 print(f'Training data size: {len(train_df)}')
 print("Extracting Data...")
 train_text, train_target = train_df['comment_text'].values, torch.from_numpy(train_df[TARGET_COLUMNS].astype(np.float32 ).values)
 train_has_identities =(train_df[identity_columns].values >= 0.5 ).any(axis=1)
 train_is_toxic =(train_df['target'].values >= 0.5)
 train_weights =(1 + ~train_is_toxic)* train_has_identities + train_is_toxic * ~train_has_identities + 1
train_weights = train_weights.astype(np.float32)/ train_weights.mean()
train_weights = torch.from_numpy(train_weights)
 del train_df, train_has_identities, train_is_toxic
gc.collect()
print("Tokenizing...")
 for index, comment in enumerate(tqdm(train_text)) :
train_text[index] = tokenize(comment)
 gc.collect()<load_from_csv>
del op, user, prd, uxp
gc.collect()
with timer('Process test.csv'):
print("Loading File...")
 test_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 print(f'Testing data size: {len(test_df)}')
 print("Extracting Data...")
 test_ids, test_text = torch.from_numpy(test_df['id'].astype(np.int32 ).values), test_df['comment_text'].values
del test_df
gc.collect()
print("Tokenizing...")
 for index, comment in enumerate(tqdm(test_text)) :
test_text[index] = tokenize(comment)
 gc.collect()<categorify>
orders_last = orders[(orders.eval_set=='train')|(orders.eval_set=='test')]
with timer('Calculating Lengths'):
oversized = 0
print('Trimming')
 for index, comment in enumerate(train_text):
if len(comment)> MAX_LEN:
train_text[index] = comment[:MAX_LEN]
oversized += 1
train_lengths = torch.tensor([len(comment)for comment in train_text], dtype=torch.int16)
 for index, comment in enumerate(test_text):
if len(comment)> MAX_LEN:
test_text[index] = comment[:MAX_LEN]
oversized += 1
test_lengths = torch.tensor([len(comment)for comment in test_text], dtype=torch.int16)
 print(f'{oversized} comment(s )({oversized*100/(len(train_text)+len(test_text)) }%)are longers than {MAX_LEN}')
 gc.collect()<count_unique_values>
data = data.merge(orders_last, on='user_id', how='left')
 data.head()
with timer('Indexing tokens'):
vocab = Counter()
print('Counting training tokens...')
 vocab.update(token for comment in tqdm(train_text)for token in comment)
 print('Counting testing tokens...')
 vocab.update(token for comment in tqdm(test_text)for token in comment)
 print(f'Full vocabulary size is {len(vocab)} covering {sum(vocab.values())} tokens.')
 print('Top 20 words:', vocab.most_common(20))
top_words, top_freq = zip(*vocab.most_common(min(len(vocab),MAX_VOCAB_SIZE)))
 print(f'Top-{len(top_words)} covers {sum(top_freq)*100/sum(vocab.values())}%.')
 del vocab
del top_freq
print("Building token index...")
 token2index = {token:index for index, token in enumerate(['<PAD>', '<UNK>'] + list(top_words)) }
del top_words
gc.collect()
print("Indexing training data...")
 train_input = torch.zeros(len(train_text), MAX_LEN, dtype=torch.int32)
 for index, comment in enumerate(tqdm(train_text)) :
train_input[index,:len(comment)] = torch.tensor([token2index.get(token,1)for token in comment], dtype=torch.int32)
 del train_text
gc.collect()
print("Indexing testing data...")
 test_input = torch.zeros(len(test_text), MAX_LEN, dtype=torch.int32)
 for index, comment in enumerate(tqdm(test_text)) :
test_input[index,:len(comment)] = torch.tensor([token2index.get(token,1)for token in comment], dtype=torch.int32)
 del test_text
gc.collect()<feature_engineering>
data_train = data[data.eval_set=='train']
data_train = data_train.merge(order_products_train, on=['product_id', 'order_id'], how='left')
 data_train = data_train.drop(['order_id','eval_set', 'add_to_cart_order'], axis=1)
 data_train = data_train.fillna(0)
 data_train.head()
with timer('Resolving zero length elements'):
zero_mask =(train_lengths == 0)
 print(f'found {zero_mask.sum().item() } zero length elements in training data')
 train_lengths[zero_mask] = 1
train_input[zero_mask, 0] = 1
zero_mask =(test_lengths == 0)
 print(f'found {zero_mask.sum().item() } zero length elements in testing data')
 test_lengths[zero_mask] = 1
test_input[zero_mask, 0] = 1<define_variables>
data_test = data[data.eval_set=='test']
data_test = data_test.drop(['eval_set', 'order_id'], axis=1)
 data_test = data_test.fillna(0)
 data_test.head()
embedding_files = [
{'file': '.. /input/pickled-crawl300d2m-for-kernel-competitions/crawl-300d-2M.pkl', 'size': 300},
{'file': '.. /input/pickled-glove840b300d-for-10sec-loading/glove.840B.300d.pkl', 'size': 300}
]<categorify>
del data
del orders_last
gc.collect()
vector_size = sum(file['size'] for file in embedding_files)
 word_embeddings = torch.empty(len(token2index), vector_size, dtype=torch.float32)
 word_embeddings[0] = 0
start = 0
for file in embedding_files:
with timer(f'Loading {file["file"]}'):
w2v = pickle.load(open(file["file"], 'rb'))
size = file['size']
end = start + size
unk_gen = lambda: F.normalize(torch.randn(size), p=2, dim=0)
 word_embeddings[1, start:end] = unk_gen()
not_found = []
for token, index in tqdm(token2index.items()):
if index < 2: continue
try:
word_embeddings[index, start:end] = torch.from_numpy(w2v[token])
 except KeyError:
word_embeddings[index, start:end] = unk_gen()
not_found.append(( index, token))
print(f'Could not find vectors for {len(not_found)} token(s )({len(not_found)*100/(len(token2index)-2)}%)')
 not_found.sort()
print('Top-10 not found words:
'+'
'.join(f'{str(index+1)}- {token}' for index, token in not_found[:10]))
del w2v
del not_found
gc.collect()
start = end<define_variables>
data_train = data_train.set_index(['user_id', 'product_id'])
 data_test = data_test.set_index(['user_id', 'product_id'] )
class LengthSortedBatchSampler(Sampler):
def __init__(self, lengths, batch_size):
self.lengths = lengths
self.batch_size = batch_size
def __iter__(self):
shuffled_indices = torch.randperm(self.lengths.size(0))
shuffled_lengths = self.lengths[shuffled_indices]
shuffled_lengths, sorted_indices = shuffled_lengths.sort()
shuffled_indices = shuffled_indices[sorted_indices]
batches = self.batch_size * torch.randperm(len(self))
for batch_start in batches:
yield shuffled_indices[batch_start:batch_start+self.batch_size]
def __len__(self):
return(len(self.lengths)+ self.batch_size - 1)// self.batch_size<create_dataframe>
data_train.loc[:, 'reordered'] = data_train.reordered.fillna(0)
 X_train, X_val, y_train, y_val = train_test_split(data_train.drop('reordered', axis=1), data_train.reordered,
test_size=0.2, random_state=42)
 del data_train
gc.collect()
d_train = xgboost.DMatrix(X_train, y_train)
 xgb_params = {
"objective" : "reg:logistic"
,"eval_metric" : "logloss"
,"eta" : 0.1
,"max_depth" : 6
,"min_child_weight" :10
,"gamma" :0.70
,"subsample" :0.76
,"colsample_bytree" :0.95
,"alpha" :2e-05
,"lambda" :10
}
watchlist= [(d_train, "train")]
bst = xgboost.train(params=xgb_params, dtrain=d_train, num_boost_round=80, evals=watchlist, verbose_eval=10)
 xgboost.plot_importance(bst )
BATCH_SIZE = 512
train_dataset = TensorDataset(train_lengths, train_input, train_target, train_weights)
 train_dataloader = DataLoader(train_dataset, batch_sampler=LengthSortedBatchSampler(train_lengths, BATCH_SIZE))
test_dataset = TensorDataset(test_lengths, test_input)
 test_dataloader = DataLoader(test_dataset, batch_size=BATCH_SIZE )<normalization>
del [X_train, X_val, y_train, y_val]
gc.collect()
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
class Model(nn.Module):
def __init__(self, embeddings, hidden_size=128, layer_count=2, linear_layer_count=2, num_outputs=len(TARGET_COLUMNS)) :
super(Model, self ).__init__()
vocab_size, emb_dim = embeddings.size()
self.embed = nn.Embedding.from_pretrained(embeddings)
 self.embed_dropout = SpatialDropout(0.2)
 self.gru = nn.GRU(emb_dim, hidden_size, num_layers=layer_count, batch_first=True, bidirectional=True)
 feats_size = 2*2*hidden_size
self.linear_layers = nn.ModuleList([nn.Sequential(
 nn.Dropout(0.5),
nn.Linear(feats_size, feats_size),
nn.PReLU() 
)for _ in range(linear_layer_count)])
 self.head = nn.Sequential(
 nn.Dropout(0.5),
nn.Linear(feats_size, num_outputs)
)
 def forward(self, sentences, lengths):
embeddings = self.embed(sentences)
 embeddings = self.embed_dropout(embeddings)
 packed_input = pack_padded_sequence(embeddings, lengths, batch_first=True, enforce_sorted=False)
 packed_output, h = self.gru(packed_input)
 output, _ = pad_packed_sequence(packed_output, batch_first=True)
 output_max, _ = output.max(dim=1)
 output_mean = output.sum(dim=1)/(lengths.unsqueeze(1 ).float() +(1e-8))
feats = torch.cat([output_max, output_mean], dim=1)
 for layer in self.linear_layers:
feats = feats + layer(feats)
 return self.head(feats )<compute_train_metric>
d_test = xgboost.DMatrix(data_test)
 data_test = data_test.reset_index()
data_test = data_test[['product_id', 'user_id']]
data_test["reordered"] =(bst.predict(d_test)> 0.21 ).astype(int)
 del bst
target_weights = torch.ones(len(TARGET_COLUMNS), dtype=torch.float32)
 target_weights[0] = len(TARGET_COLUMNS)+1
target_weights /= 2
target_weights = target_weights.cuda()
def loss_fn(predicted, target, weights):
return(( F.binary_cross_entropy_with_logits(predicted, target, reduction='none')* weights.unsqueeze(1)) * target_weights ).mean()<train_model>
orders_test = orders[orders.eval_set=='test']
def fit_one_epoch(model, optimizer, epoch):
model.train()
interactive = is_interactive()
epoch_loss = 0
batch_it = tqdm(train_dataloader, desc="Epoch {}/{}".format(epoch, num_epochs))
for batch_lengths, batch_sentences, batch_target, batch_weights in batch_it:
optimizer.zero_grad()
predicted = model(batch_sentences.cuda().long() , batch_lengths.cuda())
 loss = loss_fn(predicted, batch_target.cuda() , batch_weights.cuda())
 loss.backward()
optimizer.step()
batch_loss = loss.item()
if interactive:
batch_it.set_postfix({'batch loss': batch_loss})
 epoch_loss += batch_loss * batch_lengths.size(0)
 if interactive:
batch_it.close()
epoch_loss /= train_lengths.size(0)
 return epoch_loss<predict_on_test>
data_test = data_test.merge(orders_test[["user_id", "order_id"]], on='user_id', how='left')
 data_test.head()
def get_predictions(model):
model.eval()
test_size = test_ids.size(0)
 predictions = torch.empty(test_size, dtype=torch.float)
 index = 0
for batch_lengths, batch_sentences in tqdm(test_dataloader, desc="Test"):
predicted = torch.sigmoid(model(batch_sentences.cuda().long() , batch_lengths.cuda()))
index_to = index + predicted.size(0)
 predictions[index:index_to] = predicted[:,0].cpu().detach()
index = index_to
return predictions<choose_model_class>
del orders
del orders_test
gc.collect()
num_models = 7
predictions = torch.zeros(test_ids.size(0), dtype=torch.float)
 total_weight = 0
for index in range(1,num_models+1):
print(f'Start training model
model = Model(embeddings=word_embeddings, hidden_size=128, layer_count=2, linear_layer_count=2 ).cuda()
optimizer = optim.Adam(model.parameters() , lr=1e-3)
 lr_scheduler = optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda epoch: 0.6**epoch)
 num_epochs = 4
history = []
for epoch in range(1,num_epochs+1):
epoch_loss = fit_one_epoch(model, optimizer, epoch)
 print("Epoch {}/{}: Average Loss={}".format(epoch, num_epochs, epoch_loss), flush=True)
 history.append({'epoch': epoch, 'loss': epoch_loss})
 epoch_predictions = get_predictions(model)
 epoch_weight = 2**epoch
predictions += epoch_weight * epoch_predictions
total_weight += epoch_weight
lr_scheduler.step()
view_history(history)
 predictions /= total_weight<drop_column>
data_test['product_id'] = data_test.product_id.astype(int)
 data_test = data_test.drop('user_id', axis=1)
 gc.collect()
del identity_columns, train_target, train_weights, test_ids, train_lengths, test_lengths, token2index, train_input, test_input, zero_mask
del word_embeddings, train_dataset, train_dataloader, test_dataset, test_dataloader, target_weights, model, optimizer, lr_scheduler
del history, epoch_predictions
gc.collect()<define_variables>
d = dict()
for row in data_test.itertuples() :
if row.reordered == 1:
try:
d[row.order_id] += ' ' + str(row.product_id)
 except:
d[row.order_id] = str(row.product_id)
 for order in data_test.order_id:
if order not in d:
d[order] = 'None'
gc.collect()
package_dir = ".. /input/ppbert/pytorch-pretrained-bert/pytorch-pretrained-BERT"
sys.path.append(package_dir )<set_options>
sub = pd.DataFrame.from_dict(d, orient='index')
 sub.reset_index(inplace=True)
 sub.columns = ['order_id', 'products']
sub.to_csv('sub.csv', index=False )
<string_transform><EOS> 
submission = pd.read_csv(".. /working/sub.csv")
 submission.head()