Kaggle competition: Riiid Answer Correctness Prediction
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 ct=train.shape[0]
input_ids=np.ones(( ct,MAX_LEN),dtype="int32")
 attention_mask=np.zeros(( ct,MAX_LEN),dtype="int32")
 token_type_ids=np.zeros(( ct,MAX_LEN),dtype="int32")
 start_tokens=np.zeros(( ct,MAX_LEN),dtype="int32")
 end_tokens=np.zeros(( ct,MAX_LEN),dtype="int32" )<string_transform>
full_train = pd.read_pickle(".. /input/riiid-train-data-multiple-formats/riiid_train.pkl.gzip")
 questions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')
 lectures = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv' )
for k in range(train.shape[0]):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []
idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0:
toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1 
<load_from_csv>
full_train = full_train[['row_id','user_id','content_id','content_type_id','answered_correctly']]
train = full_train.groupby('user_id' ).tail(400)
 test = full_train.groupby('user_id' ).tail(4)
 train = train.drop(test.index )
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 ct_test = test.shape[0]
input_ids_t = np.ones(( ct_test,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct_test,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct_test,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1 
<load_pretrained>
question_average = pd.DataFrame(full_train.loc[full_train['content_type_id'] == 0].groupby(['content_id'])['answered_correctly'].mean() ).rename(columns={'answered_correctly':'question_average'})
 question_count = pd.DataFrame(full_train.loc[full_train['content_type_id'] == 0].groupby(['content_id'] ).size() ,columns=['question_count'])
 question_df = question_average.join(question_count)
 question_df['question_sum'] = question_df['question_average'] * question_df['question_count']
question_df = question_df.join(questions,how='outer')[['question_average','question_count','question_sum']]
question_df['question_average'].fillna(0,inplace=True)
 question_df['question_count'].fillna(0,inplace=True)
 question_df['question_sum'].fillna(0,inplace=True)
 del question_average,question_count
gc.collect()
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(128, 2, padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='binary_crossentropy', optimizer=optimizer)
 return model 
<predict_on_test>
train = train.join(question_df,on=['content_id'], rsuffix='_question' )
%%time
n_splits=5
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
DISPLAY=1
for i in range(5):
print('
print('
print('
K.clear_session()
model = build_model()
model.load_weights('.. /input/model4/v4-roberta-%i.h5'%i)
 print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/n_splits
preds_end += preds[1]/n_splits 
<categorify>
del full_train
gc.collect()
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st )<save_to_csv>
def reduce_memory_usage(train_data):
train_data['prior_question_had_explanation'] = train_data['prior_question_had_explanation'].fillna(False ).astype('bool')
 return train_data
test['selected_text'] = all
submission=test[['textID','selected_text']]
submission.to_csv('submission.csv',index=False)
 submission.head(5 )<import_modules>
mean_user = train.loc[train['content_type_id'] == False].groupby(['user_id'])['answered_correctly'].mean().mean()
mean_question = train.loc[train['content_type_id'] == False].groupby(['content_id'])['answered_correctly'].mean().mean()
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import tensorflow as tf<load_from_csv>
train['user_shift_question'] = train.loc[train['content_type_id'] == False].groupby(['user_id'])['question_average'].shift()
train_data = pd.read_csv('.. /input/train.csv')
 train_y = train_data["label"]
train_data.drop(["label"], axis=1, inplace=True)
 train_X = train_data
train_X = train_X.values.reshape(-1, 28, 28, 1)
 train_y = train_y.values
train_y = tf.keras.utils.to_categorical(train_y)
 train_X = train_X/255.00
test_X = pd.read_csv('.. /input/test.csv')
 test_X = test_X.values.reshape(-1,28,28,1)
 test_X = test_X / 255.0<choose_model_class>
train['user_shift_question'] = train.loc[train['content_type_id'] == False].groupby(['user_id'])['question_average'].shift()
cumulated_question = train.loc[train['content_type_id'] == False].groupby(['user_id'])['user_shift_question'].agg(['cumsum','cumcount'])
 train.loc[train['content_type_id'] == False,'average_past_questions'] = cumulated_question['cumsum'] /(cumulated_question['cumcount'] + 1)
 train.drop(columns=['user_shift_question'],inplace=True)
 train['average_past_questions'].fillna(mean_question,inplace=True)
 del cumulated_question
model = tf.keras.Sequential([
tf.keras.layers.Conv2D(32, kernel_size =(3,3), padding = 'same', activation ='relu', input_shape =(28,28,1)) ,
tf.keras.layers.Conv2D(32, kernel_size =(3,3), padding = 'same', activation ='relu'),
tf.keras.layers.Dropout(0.7),
tf.keras.layers.Conv2D(32, kernel_size =(3,3), padding = 'same', activation ='relu'),
tf.keras.layers.Conv2D(32, kernel_size =(3,3), padding = 'same', activation ='relu'),
tf.keras.layers.MaxPool2D(pool_size=(2,2), strides=2),
tf.keras.layers.Conv2D(32, kernel_size =(7,7), padding = 'same', activation ='relu'),
tf.keras.layers.Conv2D(32, kernel_size =(7,7), padding = 'same', activation ='relu'),
tf.keras.layers.MaxPool2D(pool_size=(2,2), strides=2),
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(1024, activation = "relu"),
tf.keras.layers.Dense(256, activation = "relu"),
tf.keras.layers.Dense(10, activation = "softmax")
 ] )<train_model>
train['user_shift'] = train.loc[train['content_type_id'] == False].groupby(['user_id'])['answered_correctly'].shift()
cumulated = train.loc[train['content_type_id'] == False].groupby(['user_id'])['user_shift'].agg(['cumsum', 'cumcount'])
 train.loc[train['content_type_id'] == False,'answered_correctly_user_average'] = cumulated['cumsum'] / cumulated['cumcount']
train['answered_correctly_user_average'].fillna(mean_user,inplace=True)
 train.drop(columns=['user_shift'], inplace=True)
 del cumulated
datagen = tf.keras.preprocessing.image.ImageDataGenerator(rotation_range=43, zoom_range=0.24)
 datagen.fit(train_X)
 ln_fc = lambda x: 1e-3 * 0.99 ** x
lrng_rt = tf.keras.callbacks.LearningRateScheduler(ln_fc)
 digitizer = model.fit_generator(datagen.flow(train_X, train_y, batch_size=1024), epochs=80, callbacks=[lrng_rt] )<save_to_csv>
user_average = pd.DataFrame(train.loc[train['content_type_id'] == 0].groupby(['user_id'])['answered_correctly_user_average'].last() ).rename(columns={'answered_correctly_user_average':'user_average'})
 user_count = pd.DataFrame(train.loc[train['content_type_id'] == 0].groupby(['user_id'] ).size() - 1,columns=['user_count'] )
predictions = model.predict(test_X)
 predictions[354]
pred = np.argmax(predictions, axis=1)
 plt.imshow(test_X[354][:,:,0],cmap='gray')
 plt.show()
pred[354]
pred_digits = pd.DataFrame({'ImageId': range(1,len(test_X)+1),'Label':pred })
 pred_digits.to_csv("pre_digits.csv",index=False )<set_options>
tmp = train.loc[train['content_type_id'] == False].groupby(['user_id'] ).mean()
train['performance_before'] = train['answered_correctly_user_average'] - train['average_past_questions']
user_performance = pd.DataFrame(train.loc[train['content_type_id'] == 0].groupby(['user_id'])['performance_before'].last() ).rename(columns={'performance_before':'performance'})
 del tmp
%reload_ext autoreload
%autoreload 2
%matplotlib inline<set_options>
gc.collect()
print(torch.cuda.is_available() , torch.backends.cudnn.enabled )<load_from_csv>
user_df = user_performance.join(user_average ).join(user_count)
 user_df['user_sum'] = user_df['user_average'] * user_df['user_count']
train_df = pd.read_csv(path/"train.csv")
 train_df.head()<load_from_csv>
def question_average_sum_by_user(df,question_df):
my_dict = {}
group = df.groupby(['user_id'])
 for user, val in group:
average_sum = 0.0
for row_index, row in val.iterrows() :
if(row['content_type_id'] == False):
question_id = row['content_id']
question_average = question_df.at[question_id,'question_average']
average_sum += question_average
my_dict[user] = [average_sum]
return pd.DataFrame.from_dict(my_dict,orient='index',columns=['question_average_sum'] )
test_df = pd.read_csv(path/"test.csv")
 test_df.head()<define_variables>
def add_answers_to_prior_df(current_df,prior_df):
prior_df_ = prior_df.copy()
if(prior_df.shape[0] > 0):
val = eval(current_df.iloc[0]['prior_group_answers_correct'])
 if(len(val)== prior_df.shape[0]):
prior_df_['answered_correctly_response'] = val
return prior_df_
TRAIN = Path(".. /train")
 TEST = Path(".. /test" )<train_model>
def build_question_df(prior_df,question_df):
if(prior_df.shape[0] == 0):
return question_df
question_sum_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['content_id'])['answered_correctly_response'].sum())\ 
.rename(columns={'answered_correctly_response':'question_sum'})
 question_count_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['content_id'] ).size() ,columns=['question_count'])
 question_df = question_df.join(question_sum_prior,rsuffix='_previous' ).join(question_count_prior,rsuffix='_previous')
 question_df['question_average'].fillna(0,inplace=True)
 question_df['question_count'].fillna(0,inplace=True)
 question_df['question_sum'].fillna(0,inplace=True)
 question_df['question_sum_previous'].fillna(0,inplace=True)
 question_df['question_count_previous'].fillna(0,inplace=True)
 question_df['question_sum'] = question_df['question_sum'] + question_df['question_sum_previous']
question_df['question_count'] = question_df['question_count'] + question_df['question_count_previous']
question_df['question_average'] = question_df['question_sum'] / question_df['question_count']
question_df.drop(['question_count_previous','question_sum_previous'],inplace=True,axis=1)
 return question_df
if os.path.isdir(TRAIN):
print('Train directory has been created')
 else:
print('Train directory creation failed.')
 if os.path.isdir(TEST):
print('Test directory has been created')
 else:
print('Test directory creation failed.' )<import_modules>
def build_user_df(prior_df,user_df,question_df):
if(prior_df.shape[0] == 0):
return user_df
user_sum_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['user_id'])['answered_correctly_response'].sum())\ 
.rename(columns={'answered_correctly_response':'user_sum'})
 user_count_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['user_id'] ).size() ,columns=['user_count'])
 user_df = user_df.join(user_sum_prior,how='outer',rsuffix='_previous' ).join(user_count_prior,rsuffix='_previous')
 user_df['performance'].fillna(0,inplace=True)
 user_df['user_average'].fillna(0,inplace=True)
 user_df['user_count'].fillna(0,inplace=True)
 user_df['user_sum'].fillna(0,inplace=True)
 user_df['user_count_previous'].fillna(0,inplace=True)
 user_df['user_sum_previous'].fillna(0,inplace=True)
 user_df = user_df.join(question_average_sum_by_user(prior_df,question_df))
user_df['question_average_sum'].fillna(0,inplace=True)
 user_df['user_mean_performance'] =(user_df['user_sum'] - user_df['performance'] * user_df['user_count'] + user_df['question_average_sum'])/(user_df['user_count'] + user_df['user_count_previous'])
 user_df['user_sum'] = user_df['user_sum'] + user_df['user_sum_previous']
user_df['user_count'] = user_df['user_count'] + user_df['user_count_previous']
user_df['user_average'] = user_df['user_sum'] / user_df['user_count']
user_df['performance'] = user_df['user_average'] - user_df['user_mean_performance']
user_df.drop(['user_sum_previous','user_count_previous','question_average_sum','user_mean_performance'],axis=1,inplace=True)
 return user_df
from PIL import Image<data_type_conversions>
prior_df = pd.DataFrame()
current_df = pd.DataFrame()
prior_df = add_answers_to_prior_df(current_df,prior_df)
 question_df = build_question_df(prior_df,question_df)
 user_df = build_user_df(prior_df,user_df,question_df )
def pix2img(pix_data, filepath):
img_mat = pix_data.reshape(28,28)
 img_mat = img_mat.astype(np.uint8())
 img_dat = Image.fromarray(img_mat)
 img_dat.save(filepath )<feature_engineering>
TARGET_COL = ['answered_correctly']
FEATURE_COLS = ['row_id', 'performance', 'question_average']
tfms = get_transforms(do_flip = False )<train_model>
def data_transform(df, is_training = True, is_validation = True):
df = df.join(question_df['question_average'],on=['content_id'],rsuffix='_question_average')
 df = df.join(user_df[['performance','user_average', 'user_count']],on=['user_id'],rsuffix='_right')
 df['is_beginning'] = df['user_count'] < 20
df = df.loc[df['content_type_id'] == False]
if is_training or is_validation:
df = df[FEATURE_COLS + TARGET_COL]
else:
df = df[FEATURE_COLS]
return df
print('test : ',TEST)
 print('train: ', TRAIN)
 print(type(TEST))<define_variables>
%%time
train = data_transform(train )
path =(".. /train")
<load_pretrained>
test = data_transform(test,False )
np.random.seed(42)
 data = ImageDataBunch.from_folder(path, train=".", test =(".. /test"), valid_pct=0.2,
ds_tfms=get_transforms() , size=28, num_workers=0 ).normalize(imagenet_stats )<choose_model_class>
X_train = train[FEATURE_COLS]
y_train = train[TARGET_COL]
X_test = test[FEATURE_COLS]
y_test = test[TARGET_COL]
learn = cnn_learner(data, base_arch = models.resnet34, metrics = accuracy,model_dir="/tmp/models", callback_fns=ShowGraph )<train_model>
lgb_train = lgb.Dataset(X_train.iloc[:,1:],y_train)
 lgb_val = lgb.Dataset(X_test.iloc[:,1:],y_test )
learn.fit_one_cycle(5, 1e-03 )<save_model>
del X_train,y_train
gc.collect()
learn.save('model1' )<train_model>
def LGB_bayesian(
 num_leaves,
seed, 
):
num_leaves = int(num_leaves)
 seed = int(seed)
 param = {
'objective': 'binary',
'seed': seed,
'metric': 'auc',
'learning_rate': 0.05,
'max_bin': 1000,
'num_leaves': num_leaves,
'num_iterations' : 20
}
clf = lgb.train(param, lgb_train, num_boost_round=50, valid_sets = [lgb_train, lgb_val], verbose_eval=0, early_stopping_rounds = 50)
 predictions = pd.DataFrame(clf.predict(X_test.iloc[:,1:]),index=X_test.index)
 score = roc_auc_score(y_test,predictions[0])
 return score
learn.fit_one_cycle(5 , 1e-04 )<train_model>
bounds_LGB = {
'num_leaves':(31, 500),
'seed':(0,1000)
 }
LGB_BO = BayesianOptimization(LGB_bayesian, bounds_LGB, random_state=42)
 init_points = 10
n_iter = 10
print('-' * 130)
 warnings.filterwarnings("ignore")
 with warnings.catch_warnings() :
warnings.filterwarnings('ignore')
 LGB_BO.maximize(init_points=init_points, n_iter=n_iter, acq='ucb', xi=0.0, alpha=1e-6 )
learn.fit_one_cycle(5 , slice(1e-05,1e-04))<train_model>
param_lgb = {
'objective': 'binary',
'seed': int(LGB_BO.max['params']['seed']),
'metric': 'auc',
'learning_rate': 0.05,
'max_bin': 1000,
'num_leaves': int(LGB_BO.max['params']['num_leaves']),
'num_iterations' : 20
}
model = lgb.train(
 param_lgb,
lgb_train,
valid_sets=[lgb_train,lgb_val],
verbose_eval=1,
num_boost_round=100,
early_stopping_rounds=10
)
learn.fit_one_cycle(5 , slice(1e-06,1e-05))<load_pretrained>
%%time
predictions = pd.DataFrame(model.predict(X_test.iloc[:,1:]),index=X_test.index)
np.random.seed(42)
 data = ImageDataBunch.from_folder(path, train=".", test =(".. /test"), valid_pct=0.2,
ds_tfms=get_transforms() , size=69, num_workers=0 ).normalize(imagenet_stats)
 learn.data = data
data.train_ds[0][0].shape<define_search_space>
roc_auc_score(y_test,predictions[0] )
<train_model><EOS> 
env = riiideducation.make_env()
iter_test = env.iter_test()
iter_nb = 0
for(current_df, sample_prediction_df)in iter_test:
if(iter_nb != 0):
prior_df = add_answers_to_prior_df(current_df,prior_df)
 question_df = build_question_df(prior_df,question_df)
 user_df = build_user_df(prior_df,user_df,question_df)
 prior_df = current_df.copy()
current_df = data_transform(current_df,False,False)
 current_df['answered_correctly'] = model.predict(current_df.iloc[:,1:])
 iter_nb = 1
env.predict(current_df.loc[:, ['row_id', 'answered_correctly']] )