Kaggle competition: What's Cooking? (Kernels Only)
if VALID:
valid_dataset = SAKTDataset(valid_group, n_skill, subset="valid")
 val_loader = DataLoader(valid_dataset,
batch_size=VAL_BATCH_SIZE,
shuffle=False)
 criterion = nn.BCEWithLogitsLoss()
criterion.to(device)
 val_loss, val_acc, val_auc = valid_epoch(model, val_loader, criterion, device=device)
 print(f"Valid: loss - {val_loss:.2f} acc - {val_acc:.4f} auc - {val_auc:.4f}" )<split>
recipe_data = json.loads(open('.. /input/train.json' ).read() )
env = riiideducation.make_env()
iter_test = env.iter_test()<feature_engineering>
colnames = list(unique_ing)
 data = pd.DataFrame(0, index=recipe_id, columns=colnames )
model.eval()
prev_test_df = None
for(test_df, sample_prediction_df)in iter_test:
if(prev_test_df is not None)&(psutil.virtual_memory().percent<95):
print(psutil.virtual_memory().percent)
 prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prev_test_df = prev_test_df[prev_test_df.content_type_id == False]
prev_group = prev_test_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
prev_group_content = prev_group[prev_user_id][0]
prev_group_ac = prev_group[prev_user_id][1]
if prev_user_id in group.index:
group[prev_user_id] =(np.append(group[prev_user_id][0],prev_group_content),
np.append(group[prev_user_id][1],prev_group_ac))
else:
group[prev_user_id] =(prev_group_content,prev_group_ac)
 if len(group[prev_user_id][0])>MAX_SEQ:
new_group_content = group[prev_user_id][0][-MAX_SEQ:]
new_group_ac = group[prev_user_id][1][-MAX_SEQ:]
group[prev_user_id] =(new_group_content,new_group_ac)
 prev_test_df = test_df.copy()
test_df = test_df[test_df.content_type_id == False]
test_dataset = TestDataset(group, test_df, n_skill)
 test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)
 outs = []
for item in test_dataloader:
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, _ = model(x, target_id)
 output = torch.sigmoid(output)
 output = output[:, -1]
outs.extend(output.view(-1 ).data.cpu().numpy())
 test_df['answered_correctly'] = outs
env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )<load_from_csv>
data['cuisine'] = cuisine
sns.set()
sub = pd.read_csv('.. /working/submission.csv')
 sub['answered_correctly'].hist() ;<import_modules>
for recipe in recipe_data:
index = recipe['id']
ingredients =recipe['ingredients']
for ingredient in ingredients:
data.at[index, ingredient] = 1
import gc
import json
import pandas as pd
from pathlib import Path
import sqlite3
import riiideducation
import time
import xgboost as xgb<split>
y, label = pd.factorize(data['cuisine'] )
env = riiideducation.make_env()
iter_test = env.iter_test()<define_variables>
X_train = data[colnames].values.astype(float)
 y_train = keras.utils.to_categorical(y, num_classes=20 )
PATH = Path('.. /input/riiid-submission' )<choose_model_class>
model = Sequential()
model.add(Dropout(0.3))
model.add(Dense(512, input_dim=6714, activation='linear'))
model.add(LeakyReLU(alpha=.02))
model.add(Dropout(0.5))
model.add(Dense(100, activation='relu'))
model.add(Dense(20, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='Adamax', metrics=['accuracy'] )
model = xgb.Booster(model_file=PATH/'model.xgb')
 print('model loaded' )<define_variables>
model.fit(X_train, y_train,
epochs=25,
batch_size=250)
dtypes = {
'answered_correctly': 'int8',
'answered_correctly_content_id_cumsum': 'int16',
'answered_correctly_content_id_cumsum_pct': 'int16',
'answered_correctly_cumsum': 'int16',
'answered_correctly_cumsum_pct': 'int8',
'answered_correctly_cumsum_upto': 'int8',
'answered_correctly_rollsum': 'int8',
'answered_correctly_rollsum_pct': 'int8',
'answered_incorrectly': 'int8',
'answered_incorrectly_content_id_cumsum': 'int16',
'answered_incorrectly_cumsum': 'int16',
'answered_incorrectly_rollsum': 'int8',
'bundle_id': 'uint16',
'content_id': 'int16',
'content_type_id': 'int8',
'correct_answer': 'uint8',
'lecture_id': 'uint16',
'lectures_cumcount': 'int16',
'part': 'uint8',
'part_correct_pct': 'uint8',
'prior_question_elapsed_time': 'float32',
'prior_question_elapsed_time_rollavg': 'float32',
'prior_question_had_explanation': 'bool',
'question_id': 'uint16',
'question_id_correct_pct': 'uint8',
'row_id': 'int64',
'tag': 'uint8',
'tag__0': 'uint8',
'tag__0_correct_pct': 'uint8',
'tags': 'str',
'task_container_id': 'int16',
'task_container_id_orig': 'int16',
'timestamp': 'int64',
'type_of': 'str',
'user_answer': 'int8',
'user_id': 'int32'
}
batch_cols_all = [
'user_id',
'content_id',
'row_id',
'task_container_id',
'timestamp',
'prior_question_elapsed_time',
'prior_question_had_explanation'
]
batch_cols_prior = [
'user_id',
'content_id',
'content_type_id'
]
with open(PATH/'columns.json')as cj:
test_cols = json.load(cj)
 batch_cols = ['user_id', 'content_id', 'row_id'] + [c for c in batch_cols_all if c in test_cols]
print('test_cols:')
 _ = list(map(print, test_cols))
dtypes_test = {k: v for k,v in dtypes.items() if k in test_cols}
dtypes_test = {**dtypes_test, **{'user_id': 'int32', 'content_id': 'int16'}}<load_pretrained>
test_data = json.loads(open('.. /input/test.json' ).read() )
df_users_content = pd.read_pickle(PATH/'df_users_content.pkl')
 df_users_content.head()<data_type_conversions>
test_recipe_id = []
for recipe in test_data:
test_recipe_id.append(recipe['id'])
 test_df = pd.DataFrame(0, index=test_recipe_id, columns=colnames )
df_users = df_users_content[['user_id', 'answered_correctly', 'answered_incorrectly']].groupby('user_id' ).sum().reset_index()
df_users = df_users.astype({'user_id': 'int32', 'answered_correctly': 'int16', 'answered_incorrectly': 'int16'})
 df_users.head()<load_pretrained>
ingr_checker = dict.fromkeys(colnames )
df_questions = pd.read_pickle(PATH/'df_questions.pkl')
 df_questions.head()<create_dataframe>
for recipe in test_data:
index = recipe['id']
ingredients = recipe['ingredients']
for ingredient in ingredients:
if ingredient in ingr_checker:
test_df.at[index, ingredient] = 1
%%time
chunk_size = 20000
total = len(df_users_content)
 n_chunks =(total // chunk_size + 1)
 i = 0
while i < n_chunks:
df_users_content.iloc[i * chunk_size:(i + 1)* chunk_size].to_sql('users_content', conn, method='multi', if_exists='append', index=False)
 i += 1
conn.execute('CREATE UNIQUE INDEX users_content_index ON users_content(user_id, content_id)')
 del df_users_content
gc.collect()<load_from_csv>
X_test = test_df[colnames].values.astype(float )
%%time
pd.read_sql('SELECT * from users_content LIMIT 5', conn )<drop_column>
prediction = model.predict(X_test )
%%time
chunk_size = 20000
total = len(df_users)
 n_chunks =(total // chunk_size + 1)
 i = 0
while i < n_chunks:
df_users.iloc[i * chunk_size:(i + 1)* chunk_size].to_sql('users', conn, method='multi', if_exists='append', index=False)
 i += 1
_ = conn.execute('CREATE UNIQUE INDEX users_index ON users(user_id)')
 del df_users
gc.collect()<load_from_csv>
prediction_classes = prediction.argmax(axis=-1 )
%%time
pd.read_sql('SELECT * from users LIMIT 5', conn )<create_dataframe>
label_names = label[prediction_classes]
df_output = pd.DataFrame({'id' : test_recipe_id, 'cuisine' : label_names})
 df_output.head()
<load_from_csv><EOS> 
df_output.to_csv("output.csv", header=True, index=False )