Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<sort_values>
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1
sub1.sort_values("ForecastId", inplace=True)
 sub2.sort_values("ForecastId", inplace=True)
 submit.sort_values("ForecastId", inplace=True )<compute_test_metric>
import numpy as np
import random
import pandas as pd
import joblib
import psutil
TARGETS = ["ConfirmedCases", "Fatalities"]
[np.sqrt(mean_squared_error(np.log1p(sub1[t].values), np.log1p(sub2[t].values)))for t in TARGETS]<prepare_output>
_ = np.seterr(divide='ignore', invalid='ignore' )
sub_df = sub1.copy()
for t in TARGETS:
sub_df[t] = np.expm1(np.log1p(submit[t].values)*0.2 + np.log1p(sub1[t].values)*0.3 + np.log1p(sub2[t].values)*0.5 )<save_to_csv>
data_types_dict = {
'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id':'int8',
'task_container_id': 'int16',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'bool'
}
target = 'answered_correctly'
sub_df.to_csv("submission.csv", index=False )<import_modules>
print('start read train data...')
 train_df = dt.fread('.. /input/riiid-test-answer-prediction/train.csv', columns=set(data_types_dict.keys())).to_pandas()
import plotly.graph_objects as go
import matplotlib.pyplot as plt
from tqdm import tqdm
import time
from datetime import datetime
from pathlib import Path
from sklearn import preprocessing
import keras.backend as K
from keras.models import Sequential
from keras.layers import Dense, LSTM, Dropout, GRU
from keras.callbacks import EarlyStopping
from sklearn.preprocessing import StandardScaler, MinMaxScaler<load_from_csv>
print('start handle lecture data...' )
train = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/train.csv")
 test = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/test.csv")
 train.tail()<data_type_conversions>
lectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv' )
train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'])
 train['Country_Region'] = train['Country_Region'].astype(str)
 test['Country_Region'] = test['Country_Region'].astype(str)
<feature_engineering>
lectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')
 lectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'])
 part_lectures_columns = [column for column in lectures_df.columns if column.startswith('part')]
types_of_lectures_columns = [column for column in lectures_df.columns if column.startswith('type_of_')]
EMPTY_VAL = "EMPTY_VAL"
def fillState(state, country):
if state == EMPTY_VAL: return country
return state
train['Province_State'].fillna(EMPTY_VAL, inplace=True)
 train['Province_State'] = train.loc[:, ['Province_State', 'Country_Region']].apply(lambda x : fillState(x['Province_State'], x['Country_Region']), axis=1)
 test['Province_State'].fillna(EMPTY_VAL, inplace=True)
 test['Province_State'] = test.loc[:, ['Province_State', 'Country_Region']].apply(lambda x : fillState(x['Province_State'], x['Country_Region']), axis=1 )<categorify>
train_lectures = train_df[train_df.content_type_id == True].merge(lectures_df, left_on='content_id', right_on='lecture_id', how='left' )
le = preprocessing.LabelEncoder()
train['country_encoder'] = le.fit_transform(train['Country_Region'])
 train['date_int'] = train['Date'].apply(lambda x: datetime.strftime(x, '%m%d')).astype(int)
 test['country_encoder'] = le.transform(test['Country_Region'])
 test['date_int'] = test['Date'].apply(lambda x: datetime.strftime(x, '%m%d')).astype(int )<categorify>
user_lecture_stats_part = train_lectures.groupby('user_id',as_index = False)[part_lectures_columns + types_of_lectures_columns].sum()
le = preprocessing.LabelEncoder()
train['province_encoder'] = le.fit_transform(train['Province_State'])
 test['province_encoder'] = le.transform(test['Province_State'] )<feature_engineering>
lecturedata_types_dict = {
'user_id': 'int32',
'part_1': 'int8',
'part_2': 'int8',
'part_3': 'int8',
'part_4': 'int8',
'part_5': 'int8',
'part_6': 'int8',
'part_7': 'int8',
'type_of_concept': 'int8',
'type_of_intention': 'int8',
'type_of_solving_question': 'int8',
'type_of_starter': 'int8'
}
user_lecture_stats_part = user_lecture_stats_part.astype(lecturedata_types_dict )
start_time = time.time()
country = train['Country_Region'].drop_duplicates()
train_df = train.copy()
train_df.rename(columns={'Date': 'date', 'ConfirmedCases': 'cc_cases', 'Fatalities': 'ft_cases', 'Country_Region': 'country', 'Province_State': 'province'}, inplace=True)
 lags = np.arange(1,8,1)
 with parallel_backend('threading', n_jobs = -1):
with tqdm(total = len(list(train_df['date'].unique())))as pbar:
for d in train_df['date'].drop_duplicates() :
for i in country:
province = train_df[train_df['country'] == i]['province'].drop_duplicates()
for j in province:
mask =(train_df['date'] == d)&(train_df['country'] == i)&(train_df['province'] == j)
 for lag in lags:
mask_org =(train_df['date'] ==(d - pd.Timedelta(days=lag)))&(train_df['country'] == i)&(train_df['province'] == j)
 try:
train_df.loc[mask, 'cc_cases_' + str(lag)] = train_df.loc[mask_org, 'cc_cases'].values
except:
train_df.loc[mask, 'cc_cases_' + str(lag)] = 0
try:
train_df.loc[mask, 'ft_cases_' + str(lag)] = train_df.loc[mask_org, 'ft_cases'].values
except:
train_df.loc[mask, 'ft_cases_' + str(lag)] = 0
pbar.update(1)
 print('Time spent for building features is {} minutes'.format(round(( time.time() -start_time)/60,1)) )<load_from_csv>
for column in user_lecture_stats_part.columns:
if(column !='user_id'):
user_lecture_stats_part[column] =(user_lecture_stats_part[column] > 0 ).astype('int8' )
train_df = pd.read_csv(Path('/kaggle/working/', 'train_df.csv'), index_col = 0, parse_dates = ['date'])
 train_df[train_df['country'] == 'Italy'].tail()<split>
del(train_lectures)
 gc.collect()
def split_train_val(df, val_ratio):
val_len = int(len(df)* val_ratio)
 train_set = df[:-val_len]
val_set = df[-val_len:]
return train_set, val_set<prepare_x_and_y>
user_lecture_agg = train_df.groupby('user_id')['content_type_id'].agg(['sum', 'count'])
 user_lecture_agg=user_lecture_agg.astype('int16' )
test_fixed_cols = ['ForecastId', 'Province_State', 'Country_Region', 'Date']
fixed_cols = ['Id', 'province', 'country', 'date']
output_cols = ['cc_cases', 'ft_cases']
input_cols = list(set(train_df.columns.to_list())- set(fixed_cols)- set(output_cols))
print('output columns are ', output_cols)
 print('input columns are ', input_cols)
 X = train_df[input_cols]
y = train_df[output_cols]<split>
cum = train_df.groupby('user_id')['content_type_id'].agg(['cumsum', 'cumcount'])
 cum['cumcount']=cum['cumcount']+1
train_df['user_interaction_count'] = cum['cumcount']
train_df['user_interaction_timestamp_mean'] = train_df['timestamp']/cum['cumcount']
train_df['user_lecture_sum'] = cum['cumsum']
train_df['user_lecture_lv'] = cum['cumsum'] / cum['cumcount']
train_df.user_lecture_lv=train_df.user_lecture_lv.astype('float16')
 train_df.user_lecture_sum=train_df.user_lecture_sum.astype('int16')
 train_df.user_interaction_count=train_df.user_interaction_count.astype('int16')
 train_df['user_interaction_timestamp_mean']=train_df['user_interaction_timestamp_mean']/(1000*3600)
 train_df.user_interaction_timestamp_mean=train_df.user_interaction_timestamp_mean.astype('float32')
cc_input = ['cc_cases_1', 'cc_cases_2', 'cc_cases_3', 'cc_cases_4', 'cc_cases_5', 'cc_cases_6', 'cc_cases_7', 'country_encoder', 'province_encoder', 'date_int']
ft_input = ['ft_cases_1', 'ft_cases_2', 'ft_cases_3', 'ft_cases_4', 'ft_cases_5', 'ft_cases_6', 'ft_cases_7', 'country_encoder', 'province_encoder', 'date_int']
cc_output = ['cc_cases']
ft_output = ['ft_cases']
X_cc = X[cc_input]
X_ft = X[ft_input]
y_cc = y[cc_output]
y_ft = y[ft_output]
train_X_cc, val_X_cc = split_train_val(df = X_cc, val_ratio = 0.1)
 train_y_cc, val_y_cc = split_train_val(df = y_cc, val_ratio = 0.1)
 train_X_ft, val_X_ft = split_train_val(df = X_ft, val_ratio = 0.1)
 train_y_ft, val_y_ft = split_train_val(df = y_ft, val_ratio = 0.1 )<randomize_order>
del cum
gc.collect()
idx = np.random.RandomState(seed=42 ).permutation(train_X_cc.index)
 train_X_cc = train_X_cc.reindex(idx)
 train_y_cc = train_y_cc.reindex(idx)
 train_X_ft = train_X_ft.reindex(idx)
 train_y_ft = train_y_ft.reindex(idx)
<normalization>
print('start handle train_df...' )
X_scaler_cc = MinMaxScaler()
X_train_cc = X_scaler_cc.fit_transform(train_X_cc)
 X_val_cc = X_scaler_cc.transform(val_X_cc)
 y_scaler_cc = MinMaxScaler()
y_train_cc = y_scaler_cc.fit_transform(train_y_cc)
 y_val_cc = y_scaler_cc.transform(val_y_cc )<normalization>
train_df['prior_question_had_explanation'].fillna(False, inplace=True)
 train_df = train_df.astype(data_types_dict)
 train_df = train_df[train_df[target] != -1].reset_index(drop=True )
X_scaler_ft = MinMaxScaler()
X_train_ft = X_scaler_ft.fit_transform(train_X_ft)
 X_val_ft = X_scaler_ft.transform(val_X_ft)
 y_scaler_ft = MinMaxScaler()
y_train_ft = y_scaler_ft.fit_transform(train_y_ft)
 y_val_ft = y_scaler_ft.transform(val_y_ft )<data_type_conversions>
content_explation_agg=train_df[["content_id","prior_question_had_explanation",target]].groupby(["content_id","prior_question_had_explanation"])[target].agg(['mean'] )
X_train_cc = train_X_cc.to_numpy()
X_val_cc = val_X_cc.to_numpy()
X_train_ft = train_X_ft.to_numpy()
X_val_ft = val_X_ft.to_numpy()
y_train_cc = train_y_cc.to_numpy()
y_val_cc = val_y_cc.to_numpy()
y_train_ft = train_y_ft.to_numpy()
y_val_ft = val_y_ft.to_numpy()<compute_test_metric>
content_explation_agg=content_explation_agg.unstack()
content_explation_agg=content_explation_agg.reset_index()
content_explation_agg.columns = ['content_id', 'content_explation_false_mean','content_explation_true_mean']
def root_mean_squared_log_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(K.log(y_pred + 1)- K.log(y_true + 1))))<choose_model_class>
content_explation_agg.content_id=content_explation_agg.content_id.astype('int16')
 content_explation_agg.content_explation_false_mean=content_explation_agg.content_explation_false_mean.astype('float16')
 content_explation_agg.content_explation_true_mean=content_explation_agg.content_explation_true_mean.astype('float16' )
def GRU_model(n_1, input_dim, output_dim):
model = Sequential()
model.add(GRU(n_1,input_shape=(1, input_dim), activation='relu'))
model.add(Dropout(0.1))
model.add(Dense(output_dim, activation='relu'))
model.compile(loss=root_mean_squared_log_error, optimizer='adam')
 print(model.summary())
 return model<choose_model_class>
print('start handle attempt_no...' )
model_cc = GRU_model(4, X_train_cc.shape[-1], y_train_cc.shape[-1])
 model_ft = GRU_model(4, X_train_ft.shape[-1], y_train_ft.shape[-1])
 early_stop = EarlyStopping(monitor='loss', patience=5, verbose=0, mode='min' )<train_model>
train_df["attempt_no"] = 1
train_df.attempt_no=train_df.attempt_no.astype('int8')
 attempt_no_agg=train_df.groupby(["user_id","content_id"])["attempt_no"].agg(['sum'] ).astype('int8')
 train_df["attempt_no"] = train_df[["user_id","content_id",'attempt_no']].groupby(["user_id","content_id"])["attempt_no"].cumsum()
print('Start model training')
 start_time = time.time()
history_cc = model_cc.fit(X_train_cc, y_train_cc, epochs = 100,validation_data =(X_val_cc, y_val_cc), verbose = 2, callbacks=[early_stop])
 model_cc.save("model_cc.h5")
 print('Time spent for model training is {} minutes'.format(round(( time.time() -start_time)/60,1)) )<train_model>
print('start handle timestamp...')
 prior_question_elapsed_time_mean=train_df['prior_question_elapsed_time'].mean()
train_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True )
print('Start model training')
 start_time = time.time()
history_ft = model_ft.fit(X_train_ft, y_train_ft, epochs = 100,validation_data =(X_val_ft, y_val_ft), verbose = 2, callbacks=[early_stop])
 model_ft.save("model_ft.h5")
 print('Time spent for model training is {} minutes'.format(round(( time.time() -start_time)/60,1)) )<predict_on_test>
max_timestamp_u = train_df[['user_id','timestamp']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u.columns = ['user_id', 'max_time_stamp']
max_timestamp_u.user_id=max_timestamp_u.user_id.astype('int32' )
yhat_val_cc = model_cc.predict(X_val_cc)
 print(yhat_val_cc )<predict_on_test>
train_df['lagtime'] = train_df.groupby('user_id')['timestamp'].shift()
max_timestamp_u2 = train_df[['user_id','lagtime']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u2.columns = ['user_id', 'max_time_stamp2']
max_timestamp_u2.user_id=max_timestamp_u2.user_id.astype('int32' )
yhat_val_ft = model_cc.predict(X_val_ft)
 print(yhat_val_ft )<data_type_conversions>
train_df['lagtime']=train_df['timestamp']-train_df['lagtime']
lagtime_mean=train_df['lagtime'].mean()
train_df['lagtime'].fillna(lagtime_mean, inplace=True )
start_time = time.time()
test['Country_Region'] = test['Country_Region'].astype(str)
 test['Province_State'] = test['Province_State'].astype(str)
 country = test['Country_Region'].drop_duplicates()
adj_input_cols = [e for e in input_cols if e not in('province_encoder', 'country_encoder', 'date_int')]
test_df = test.copy().join(pd.DataFrame(columns = adj_input_cols + output_cols))
test_df.rename(columns={'Date': 'date', 'Country_Region': 'country', 'Province_State': 'province'}, inplace=True)
 lags = np.arange(1,8,1)
 test_overlap_mask =(test_df['date'] <= train_df['date'].max())
 train_overlap_mask =(train_df['date'] >= test_df['date'].min())
 test_df.loc[test_overlap_mask, input_cols + output_cols] = train_df.loc[train_overlap_mask, input_cols + output_cols].values
pred_dt_range = pd.date_range(start = train_df['date'].max() + pd.Timedelta(days=1), end = test_df['date'].max() , freq = '1D')
 with tqdm(total = len(pred_dt_range)) as pbar:
for d in pred_dt_range:
for i in country:
province = test_df[test_df['country'] == i]['province'].drop_duplicates()
for j in province:
mask =(test_df['date'] == d)&(test_df['country'] == i)&(test_df['province'] == j)
 for lag in lags:
mask_org =(test_df['date'] ==(d - pd.Timedelta(days=lag)))&(test_df['country'] == i)&(test_df['province'] == j)
 try:
test_df.loc[mask, 'cc_cases_' + str(lag)] = test_df.loc[mask_org, 'cc_cases'].values
except:
test_df.loc[mask, 'cc_cases_' + str(lag)] = 0
try:
test_df.loc[mask, 'ft_cases_' + str(lag)] = test_df.loc[mask_org, 'ft_cases'].values
except:
test_df.loc[mask, 'ft_cases_' + str(lag)] = 0
test_X = test_df.loc[mask, input_cols]
test_X_cc = test_X[cc_input]
X_test_cc= test_X_cc
X_test_cc = X_test_cc.to_numpy().reshape(X_test_cc.shape[0], 1, X_test_cc.shape[1])
 next_cc = model_cc.predict(X_test_cc)
 next_cc_scaled = next_cc
test_X_ft = test_X[ft_input]
X_test_ft = test_X_ft
X_test_ft = X_test_ft.to_numpy().reshape(X_test_ft.shape[0], 1, X_test_ft.shape[1])
 next_ft = model_cc.predict(X_test_ft)
 next_ft_scaled = next_ft
test_df.loc[mask, 'cc_cases'] = next_cc_scaled
test_df.loc[mask, 'ft_cases'] = next_ft_scaled
pbar.update(1)
 print('Time spent for building features is {} minutes'.format(round(( time.time() -start_time)/60,1)) )<create_dataframe>
train_df['lagtime']=train_df['lagtime']/(1000*3600)
 train_df.lagtime=train_df.lagtime.astype('float32' )
submission = pd.DataFrame()
submission['ForecastId'] = test_df['ForecastId']
submission['ConfirmedCases'] = test_df['cc_cases']
submission['Fatalities'] = test_df['ft_cases']<save_to_csv>
submission.to_csv("submission.csv",index=False )<install_modules>
train_df['lagtime2'] = train_df.groupby('user_id')['timestamp'].shift(2)
 max_timestamp_u3 = train_df[['user_id','lagtime2']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u3.columns = ['user_id', 'max_time_stamp3']
max_timestamp_u3.user_id=max_timestamp_u3.user_id.astype('int32')
 train_df['lagtime2']=train_df['timestamp']-train_df['lagtime2']
lagtime_mean2=train_df['lagtime2'].mean()
train_df['lagtime2'].fillna(lagtime_mean2, inplace=True)
!pip install skorch<set_options>
train_df['lagtime2']=train_df['lagtime2']/(1000*3600)
 train_df.lagtime2=train_df.lagtime2.astype('float32' )
warnings.filterwarnings('ignore' )<set_options>
train_df['lagtime3'] = train_df.groupby('user_id')['timestamp'].shift(3)
 train_df['lagtime3']=train_df['timestamp']-train_df['lagtime3']
lagtime_mean3=train_df['lagtime3'].mean()
train_df['lagtime3'].fillna(lagtime_mean3, inplace=True)
 train_df['lagtime3']=train_df['lagtime3']/(1000*3600)
 train_df.lagtime3=train_df.lagtime3.astype('float32' )
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed = 1234
seed_everything(seed )<load_from_csv>
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv' )<feature_engineering>
train_df['timestamp']=train_df['timestamp']/(1000*3600)
 train_df.timestamp=train_df.timestamp.astype('float16' )
def title_extract(df):
df['Title'] = df['Name'].str.extract('([a-zA-Z]+)\.')
 df['Title'] = df['Title'].apply(lambda x: 'Unknown' if x not in ['Miss','Master','Mr','Mrs', 'Dr', 'Rev'] else x)
 return df<drop_column>
train_df['delta_prior_question_elapsed_time'] = train_df.groupby('user_id')['prior_question_elapsed_time'].shift()
train_df['delta_prior_question_elapsed_time']=train_df['prior_question_elapsed_time']-train_df['delta_prior_question_elapsed_time']
train = title_extract(train)
 test = title_extract(test )<categorify>
delta_prior_question_elapsed_time_mean=train_df['delta_prior_question_elapsed_time'].mean()
train_df['delta_prior_question_elapsed_time'].fillna(delta_prior_question_elapsed_time_mean, inplace=True)
 train_df.delta_prior_question_elapsed_time=train_df.delta_prior_question_elapsed_time.astype('int32' )
def fill_nan_age(df):
df['group_mean_age'] = round(df.groupby(['Sex', 'Title'])['Age'].transform('mean'))
df['Age'].fillna(df['group_mean_age'], inplace=True)
 del df['group_mean_age']
return df<correct_missing_values>
train_df['lag'] = train_df.groupby('user_id')[target].shift()
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 user_agg = train_df.groupby('user_id')['lag'].agg(['sum', 'count'] ).astype('int16')
 cum['cumsum'].fillna(0, inplace=True)
 train_df['user_correctness'] = cum['cumsum'] / cum['cumcount']
train_df['user_correct_count'] = cum['cumsum']
train_df['user_uncorrect_count'] = cum['cumcount']-cum['cumsum']
train_df.drop(columns=['lag'], inplace=True)
 train_df['user_correctness'].fillna(0.67, inplace=True)
 train_df.user_correctness=train_df.user_correctness.astype('float16')
 train_df.user_correct_count=train_df.user_correct_count.astype('int16')
 train_df.user_uncorrect_count=train_df.user_uncorrect_count.astype('int16')
train = fill_nan_age(train)
 test = fill_nan_age(test )<feature_engineering>
del cum
gc.collect()
def alone_family(df):
df['Family'] = train['SibSp'] + train['Parch']
df['Alone'] = pd.Series(np.where(df['Family'] == 0, 1, 0))
return df<concatenate>
train = alone_family(train)
 test = alone_family(test )<concatenate>
train_df.prior_question_had_explanation=train_df.prior_question_had_explanation.astype('int8')
 explanation_agg = train_df.groupby('user_id')['prior_question_had_explanation'].agg(['sum', 'count'])
 explanation_agg=explanation_agg.astype('int16')
train = fare_cat(train)
 test = fare_cat(test )<categorify>
cum = train_df.groupby('user_id')['prior_question_had_explanation'].agg(['cumsum', 'cumcount'])
 cum['cumcount']=cum['cumcount']+1
train_df['explanation_mean'] = cum['cumsum'] / cum['cumcount']
train_df['explanation_true_count'] = cum['cumsum']
train_df['explanation_false_count'] = cum['cumcount']-cum['cumsum']
train_df.explanation_mean=train_df.explanation_mean.astype('float16')
 train_df.explanation_true_count=train_df.explanation_true_count.astype('int16')
 train_df.explanation_false_count=train_df.explanation_false_count.astype('int16' )
def data_clean(drop_col, dummies, df):
df = df.drop(drop_col, axis=1)
 df = pd.get_dummies(data=df, columns=dummies)
 return df<define_variables>
del cum
gc.collect()
col_drop = ['PassengerId', 'Name', 'Ticket', 'Cabin']
col_dummies = ['Sex', 'Embarked', 'age_cat', 'Fare_cat', 'Pclass', 'Title']<drop_column>
content_agg = train_df.groupby('content_id')[target].agg(['sum', 'count','var'])
 task_container_agg = train_df.groupby('task_container_id')[target].agg(['sum', 'count','var'])
 content_agg=content_agg.astype('float32')
 task_container_agg=task_container_agg.astype('float32' )
train_clean = data_clean(col_drop, col_dummies, train)
 train_clean.head()<drop_column>
train_df['task_container_uncor_count'] = train_df['task_container_id'].map(task_container_agg['count']-task_container_agg['sum'] ).astype('int32')
 train_df['task_container_cor_count'] = train_df['task_container_id'].map(task_container_agg['sum'] ).astype('int32')
 train_df['task_container_std'] = train_df['task_container_id'].map(task_container_agg['var'] ).astype('float16')
 train_df['task_container_correctness'] = train_df['task_container_id'].map(task_container_agg['sum'] / task_container_agg['count'])
 train_df.task_container_correctness=train_df.task_container_correctness.astype('float16' )
test_clean = data_clean(col_drop, col_dummies, test)
 test_clean.head()<prepare_x_and_y>
content_elapsed_time_agg=train_df.groupby('content_id')['prior_question_elapsed_time'].agg(['mean'])
 content_had_explanation_agg=train_df.groupby('content_id')['prior_question_had_explanation'].agg(['mean'] )
X = train_clean.drop('Survived', axis=1)
 y = train_clean['Survived']<prepare_x_and_y>
print('start questions data...' )
X = np.array(X, dtype='float32')
 y = np.array(y, dtype='float32' )<choose_model_class>
questions_df = pd.read_csv(
 '.. /input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 1,3,4],
dtype={'question_id': 'int16','bundle_id': 'int16', 'part': 'int8','tags': 'str'}
)
class TitanicModel(nn.Module):
def __init__(self, neurons=10, dropout=0.2):
super(TitanicModel, self ).__init__()
self.dense0 = nn.Linear(X.shape[1], neurons)
 self.activation0 = nn.ReLU()
self.dropout0 = nn.Dropout(dropout)
 self.dense1 = nn.Linear(neurons, neurons)
 self.activation1 = nn.ReLU()
self.dropout1 = nn.Dropout(dropout)
 self.dense2 = nn.Linear(neurons, 1)
 self.output = nn.Sigmoid()
def forward(self, x):
x = self.dense0(x)
 x = self.activation0(x)
 x = self.dropout0(x)
 x = self.dense1(x)
 x = self.activation1(x)
 x = self.dropout1(x)
 x = self.dense2(x)
 x = self.output(x)
 return x<choose_model_class>
bundle_agg = questions_df.groupby('bundle_id')['question_id'].agg(['count'] )
model = NeuralNetBinaryClassifier(module=TitanicModel,
lr = 0.001,
optimizer__weight_decay = 0.001,
verbose=0,
train_split=False,
device='cuda' )<define_search_space>
questions_df['content_sub_bundle'] = questions_df['bundle_id'].map(bundle_agg['count'] ).astype('int8' )
params = {'batch_size': [10],
'max_epochs': [25],
'optimizer': [torch.optim.Adam],
'lr': [0.01, 0.001],
'criterion': [nn.BCELoss],
'module__neurons': [10, 15, 20, 25],
'module__dropout': [0, 0.2]} 
<train_on_grid>
questions_df['tags'].fillna('188', inplace=True )
%%time
model_grid = GridSearchCV(estimator=model, param_grid=params,
scoring = 'accuracy', cv=3, verbose=0)
 model_grid = model_grid.fit(X, y )<find_best_params>
def gettags(tags,num):
tags_splits=tags.split(" ")
 result=''
for t in tags_splits:
x=int(t)
 if(x<32*(num+1)and x>=32*num):
result=result+' '+t
return result
print(f'Accuracy: {model_grid.best_score_ * 100 :.2f}%')
 print(model_grid.best_params_ )<prepare_x_and_y>
for num in range(0,6):
questions_df["tags"+str(num)] = questions_df["tags"].apply(lambda row: gettags(row,num))
le = LabelEncoder()
le.fit(np.unique(questions_df['tags'+str(num)].values))
questions_df['tags'+str(num)]=questions_df[['tags'+str(num)]].apply(le.transform )
test_clean = np.array(test_clean, dtype='float32' )<predict_on_test>
questions_df_dict = {
'tags0': 'int8',
'tags1': 'int8',
'tags2': 'int8',
'tags3': 'int8',
'tags4': 'int8',
'tags5': 'int8',
}
questions_df = questions_df.astype(questions_df_dict )
pred = model_grid.predict(test_clean )<prepare_output>
questions_df.drop(columns=['tags'], inplace=True )
submission = pd.DataFrame({'PassengerId': test['PassengerId'], 'Survived': pred})
 submission.head()<save_to_csv>
questions_df['part_bundle_id']=questions_df['part']*100000+questions_df['bundle_id']
questions_df.part_bundle_id=questions_df.part_bundle_id.astype('int32')
submission.to_csv('submission.csv', index=False )<save_to_csv>
submission.to_csv('submission.csv', index=False )<set_options>
questions_df.rename(columns={'question_id':'content_id'}, inplace=True )
%matplotlib inline
np.random.seed(2)
 dense_regularizer = L1L2(l2=0.0001)
 os.chdir('/kaggle/working')
 sns.set(style='white', context='notebook', palette='deep' )<load_from_csv>
questions_df = pd.merge(questions_df, content_explation_agg, on='content_id', how='left',right_index=True)
train = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 test = pd.read_csv("/kaggle/input/digit-recognizer/test.csv" )<prepare_x_and_y>
del content_explation_agg
Y = train["label"]
X = train.drop(labels = ["label"],axis = 1 )<feature_engineering>
questions_df['content_correctness'] = questions_df['content_id'].map(content_agg['sum'] / content_agg['count'])
 questions_df.content_correctness=questions_df.content_correctness.astype('float16')
 questions_df['content_correctness_std'] = questions_df['content_id'].map(content_agg['var'])
 questions_df.content_correctness_std=questions_df.content_correctness_std.astype('float16')
 questions_df['content_uncorrect_count'] = questions_df['content_id'].map(content_agg['count']-content_agg['sum'] ).astype('int32')
 questions_df['content_correct_count'] = questions_df['content_id'].map(content_agg['sum'] ).astype('int32' )
X = X / 255.0
test = test / 255.0<categorify>
questions_df['content_elapsed_time_mean'] = questions_df['content_id'].map(content_elapsed_time_agg['mean'])
 questions_df.content_elapsed_time_mean=questions_df.content_elapsed_time_mean.astype('float16')
 questions_df['content_had_explanation_mean'] = questions_df['content_id'].map(content_had_explanation_agg['mean'])
 questions_df.content_had_explanation_mean=questions_df.content_had_explanation_mean.astype('float16' )
Y = to_categorical(Y, num_classes = 10 )<split>
del content_elapsed_time_agg
del content_had_explanation_agg
gc.collect()
random_seed = 2
X_train, X_val, Y_train, Y_val = train_test_split(X, Y, test_size = 0.1, random_state=random_seed )<choose_model_class>
part_agg = questions_df.groupby('part')['content_correctness'].agg(['mean', 'var'])
 questions_df['part_correctness_mean'] = questions_df['part'].map(part_agg['mean'])
 questions_df['part_correctness_std'] = questions_df['part'].map(part_agg['var'])
 questions_df.part_correctness_mean=questions_df.part_correctness_mean.astype('float16')
 questions_df.part_correctness_std=questions_df.part_correctness_std.astype('float16' )
model1 = Sequential()
model1.add(Conv2D(filters = 32, kernel_size =(5,5), padding = 'Same', activation ='relu', input_shape =(28,28,1)))
 model1.add(Conv2D(filters = 32, kernel_size =(5,5), padding = 'Same', activation ='relu'))
model1.add(MaxPool2D(pool_size=(2,2)))
 model1.add(Dropout(0.25))
model1.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model1.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model1.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model1.add(Dropout(0.25))
model1.add(Flatten())
 model1.add(Dense(256, activation = "relu"))
model1.add(Dropout(0.5))
model1.add(Dense(10, activation = "softmax"))
model1.compile(optimizer = 'adam' , loss = "categorical_crossentropy", metrics=["accuracy"] )<train_model>
part_agg = questions_df.groupby('part')['content_uncorrect_count'].agg(['sum'])
 questions_df['part_uncor_count'] = questions_df['part'].map(part_agg['sum'] ).astype('int32')
 part_agg = questions_df.groupby('part')['content_correct_count'].agg(['sum'])
 questions_df['part_cor_count'] = questions_df['part'].map(part_agg['sum'] ).astype('int32' )
epochs = 5
batch_size = 86
model1.fit(
 X_train,
Y_train,
batch_size=batch_size,
epochs=epochs,
validation_data=(X_val, Y_val),
verbose=2
)<save_to_csv>
bundle_agg = questions_df.groupby('bundle_id')['content_correctness'].agg(['mean'])
 questions_df['bundle_correctness_mean'] = questions_df['bundle_id'].map(bundle_agg['mean'])
 questions_df.bundle_correctness_mean=questions_df.bundle_correctness_mean.astype('float16')
results = model1.predict(test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label")
 submission1 = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission1.to_csv("sub1.csv",index=False )<choose_model_class>
model1.compile(optimizer = 'RMSprop' , loss = "categorical_crossentropy", metrics=["accuracy"] )<choose_model_class>
del content_agg
del bundle_agg
del part_agg
gc.collect()
learning_rate_reduction = ReduceLROnPlateau(monitor='val_loss',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )<init_hyperparams>
features_dict = {
'timestamp':'float16',
'user_interaction_count':'int16',
'user_interaction_timestamp_mean':'float32',
'lagtime':'float32',
'lagtime2':'float32',
'lagtime3':'float32',
'content_id':'int16',
'task_container_id':'int16',
'user_lecture_sum':'int16',
'user_lecture_lv':'float16',
'prior_question_elapsed_time':'float32',
'delta_prior_question_elapsed_time':'int32',
'user_correctness':'float16',
'user_uncorrect_count':'int16',
'user_correct_count':'int16',
'content_correctness_std':'float16',
'content_correct_count':'int32',
'content_uncorrect_count':'int32',
'content_elapsed_time_mean':'float16',
'content_had_explanation_mean':'float16',
'content_explation_false_mean':'float16',
'content_explation_true_mean':'float16',
'task_container_correctness':'float16',
'task_container_std':'float16',
'task_container_cor_count':'int32',
'task_container_uncor_count':'int32',
'attempt_no':'int8',
'part':'int8',
'part_correctness_mean':'float16',
'part_correctness_std':'float16',
'part_uncor_count':'int32',
'part_cor_count':'int32',
'tags0': 'int8',
'tags1': 'int8',
'tags2': 'int8',
'tags3': 'int8',
'tags4': 'int8',
'tags5': 'int8',
'part_bundle_id':'int32',
'content_sub_bundle':'int8',
'prior_question_had_explanation':'int8',
'explanation_mean':'float16',
'explanation_false_count':'int16',
'explanation_true_count':'int16',
}
categorical_columns= [
'content_id',
'task_container_id',
'part',
'tags0',
'tags1',
'tags2',
'tags3',
'tags4',
'tags5',
'part_bundle_id',
'content_sub_bundle',
'prior_question_had_explanation',
]
features=list(features_dict.keys())
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )<train_model>
flag_lgbm=True
clfs = list()
params = {
'num_leaves': 200,
'max_bin':450,
'feature_fraction': 0.52,
'bagging_fraction': 0.52,
'objective': 'binary',
'learning_rate': 0.05,
"boosting_type": "gbdt",
"metric": 'auc',
}
trains=list()
valids=list()
num=1
for i in range(0,num):
train_df_clf=train_df[1200*10000:2*1400*10000]
print('sample end')
 del train_df
users=train_df_clf['user_id'].drop_duplicates()
users=users.sample(frac=0.08)
 users_df=pd.DataFrame()
users_df['user_id']=users.values
valid_df_newuser = pd.merge(train_df_clf, users_df, on=['user_id'], how='inner',right_index=True)
 del users_df
del users
gc.collect()
train_df_clf.drop(valid_df_newuser.index, inplace=True)
 print('pd.merge(train_df_clf, questions_df)')
 train_df_clf = pd.merge(train_df_clf, questions_df, on='content_id', how='left',right_index=True)
 valid_df_newuser = pd.merge(valid_df_newuser, questions_df, on='content_id', how='left',right_index=True)
 print('valid_df')
 valid_df=train_df_clf.sample(frac=0.1)
 train_df_clf.drop(valid_df.index, inplace=True)
 valid_df = valid_df.append(valid_df_newuser)
 del valid_df_newuser
gc.collect()
trains.append(train_df_clf)
 valids.append(valid_df)
 print('train_df_clf length：',len(train_df_clf))
print('valid_df length：',len(valid_df))
epochs = 5
batch_size = 86
history = model1.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),
epochs = epochs, validation_data =(X_val,Y_val),
verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size
, callbacks=[learning_rate_reduction] )<save_to_csv>
del train_df_clf
del valid_df
gc.collect()
results = model1.predict(test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label")
 submission2 = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission2.to_csv("sub2.csv",index=False )<choose_model_class>
for i in range(0,num):
X_train_np = trains[i][features].values.astype(np.float32)
 X_valid_np = valids[i][features].values.astype(np.float32)
 tr_data = lgb.Dataset(X_train_np, label=trains[i][target], feature_name=list(features))
va_data = lgb.Dataset(X_valid_np, label=valids[i][target], feature_name=list(features))
del trains
del valids
del X_train_np
del X_valid_np
gc.collect()
model1 = joblib.load('.. /input/5-start-777/lgbm775_model.pkl')
 model2 = joblib.load('.. /input/777-val-lgbm/lgbm775_model.pkl')
 model3 = joblib.load('.. /input/3-start-1200/lgbm775_model.pkl')
 clfs.append(model1)
 clfs.append(model2)
 clfs.append(model3)
 del model1
del model2
del model3
del tr_data
del va_data
gc.collect()
def Model_2(x=None):
model = Sequential()
model.add(Conv2D(64,(5, 5), input_shape=(28,28,1), padding='same', kernel_regularizer=dense_regularizer,kernel_initializer="he_normal"))
model.add(BatchNormalization())
 model.add(Activation('elu'))
model.add(Conv2D(64,(5, 5), padding='same', kernel_regularizer=dense_regularizer,kernel_initializer="he_normal"))
model.add(BatchNormalization())
 model.add(Activation('elu'))
model.add(Conv2D(64,(5, 5), padding='same', kernel_regularizer=dense_regularizer,kernel_initializer="he_normal"))
model.add(BatchNormalization())
 model.add(Activation('elu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
 model.add(Dropout(0.3))
model.add(Conv2D(128,(3, 3), padding='same', kernel_regularizer=dense_regularizer,kernel_initializer="he_normal"))
model.add(BatchNormalization())
 model.add(Activation('elu'))
model.add(Conv2D(128,(3, 3), padding='same', kernel_regularizer=dense_regularizer,kernel_initializer="he_normal"))
model.add(BatchNormalization())
 model.add(Activation('elu'))
model.add(Conv2D(128,(3, 3), padding='same', kernel_regularizer=dense_regularizer,kernel_initializer="he_normal"))
model.add(BatchNormalization())
 model.add(Activation('elu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
 model.add(Dropout(0.2))
model.add(Conv2D(256,(3, 3), padding='same', kernel_regularizer=dense_regularizer,kernel_initializer="he_normal"))
model.add(BatchNormalization())
 model.add(Activation('elu'))
model.add(Conv2D(256,(3, 3), padding='same', kernel_regularizer=dense_regularizer,kernel_initializer="he_normal"))
model.add(BatchNormalization())
 model.add(Activation('elu'))
model.add(Conv2D(256,(3, 3), padding='same', kernel_regularizer=dense_regularizer,kernel_initializer="he_normal"))
model.add(BatchNormalization())
 model.add(Activation('elu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
 model.add(Dropout(0.2))
model.add(Conv2D(512,(3, 3), padding='same', kernel_regularizer=dense_regularizer,kernel_initializer="he_normal"))
model.add(BatchNormalization())
 model.add(Activation('elu'))
model.add(Conv2D(512,(3, 3), padding='same', kernel_regularizer=dense_regularizer,kernel_initializer="he_normal"))
model.add(BatchNormalization())
 model.add(Activation('elu'))
model.add(MaxPooling2D(pool_size=(3, 3), strides=(3, 3)))
 model.add(Flatten())
 model.add(Dense(10, activation='softmax', kernel_regularizer=dense_regularizer,kernel_initializer="he_normal"))
return model
model2 = Model_2()<choose_model_class>
MAX_SEQ = 280
ACCEPTED_USER_CONTENT_SIZE = 7
EMBED_SIZE = 128
BATCH_SIZE = 64
DROPOUT = 0.1
class FFN(nn.Module):
def __init__(self, state_size = 200, forward_expansion = 1, bn_size=MAX_SEQ - 1, dropout=0.2):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, forward_expansion * state_size)
 self.relu = nn.ReLU()
self.bn = nn.BatchNorm1d(bn_size)
 self.lr2 = nn.Linear(forward_expansion * state_size, state_size)
 self.dropout = nn.Dropout(dropout)
 def forward(self, x):
x = self.relu(self.lr1(x))
x = self.bn(x)
 x = self.lr2(x)
 return self.dropout(x)
 FFN()
optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)
 model2.compile(optimizer = optimizer , loss = "sparse_categorical_crossentropy", metrics=["accuracy"] )<prepare_x_and_y>
def future_mask(seq_length):
future_mask =(np.triu(np.ones([seq_length, seq_length]), k = 1)).astype('bool')
 return torch.from_numpy(future_mask)
 future_mask(5 )
Y = train["label"]
Y = np.array(Y )<train_model>
class TransformerBlock(nn.Module):
def __init__(self, embed_dim, heads = 8, dropout = DROPOUT, forward_expansion = 1):
super(TransformerBlock, self ).__init__()
self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=heads, dropout=dropout)
 self.dropout = nn.Dropout(dropout)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim, forward_expansion = forward_expansion, dropout=dropout)
 self.layer_normal_2 = nn.LayerNorm(embed_dim)
 def forward(self, value, key, query, att_mask):
att_output, att_weight = self.multi_att(value, key, query, attn_mask=att_mask)
 att_output = self.dropout(self.layer_normal(att_output + value))
att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.dropout(self.layer_normal_2(x + att_output))
return x.squeeze(-1), att_weight
class Encoder(nn.Module):
def __init__(self, n_skill, max_seq=280, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, num_layers=1, heads = 8):
super(Encoder, self ).__init__()
self.n_skill, self.embed_dim = n_skill, embed_dim
self.embedding = nn.Embedding(2 * n_skill + 1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.layers = nn.ModuleList([TransformerBlock(embed_dim, forward_expansion = forward_expansion)for _ in range(num_layers)])
 self.dropout = nn.Dropout(dropout)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = self.dropout(x + pos_x)
 x = x.permute(1, 0, 2)
 e = self.e_embedding(question_ids)
 e = e.permute(1, 0, 2)
 for layer in self.layers:
att_mask = future_mask(e.size(0)).to(device)
 x, att_weight = layer(e, x, x, att_mask=att_mask)
 x = x.permute(1, 0, 2)
 x = x.permute(1, 0, 2)
 return x, att_weight
class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=280, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, enc_layers=1, heads = 8):
super(SAKTModel, self ).__init__()
self.encoder = Encoder(n_skill, max_seq, embed_dim, dropout, forward_expansion, num_layers=enc_layers)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
x, att_weight = self.encoder(x, question_ids)
 x = self.pred(x)
 return x.squeeze(-1), att_weight
skills = joblib.load("/kaggle/input/riiid-sakt-model-dataset-public/skills.pkl.zip")
 n_skill = len(skills)
 group = joblib.load(".. /input/riiid-sakt-model-dataset-public/group.pkl.zip")
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 def create_model() :
return SAKTModel(n_skill, max_seq=MAX_SEQ, embed_dim=EMBED_SIZE, forward_expansion=1, enc_layers=1, heads=8, dropout=0.1)
 nn_model = create_model()
nn_model
try:
nn_model.load_state_dict(torch.load(".. /input/real-777/sakt.pth"))
except:
nn_model.load_state_dict(torch.load(".. /input/real-777/sakt.pth", map_location='cpu'))
nn_model.to(device)
 nn_model.eval()
k_fold = StratifiedKFold(n_splits=10, random_state=12, shuffle=True)
 batch_size = 86
epochs = 10
for k_train_index, k_test_index in k_fold.split(X, Y):
model2.fit_generator(datagen.flow(X[k_train_index,:],Y[k_train_index], batch_size=batch_size),
epochs = epochs, validation_data =(X[k_test_index,:],Y[k_test_index]),
verbose = 2, steps_per_epoch=X[k_train_index,:].shape[0] // batch_size
, callbacks=[learning_rate_reduction] )<compute_test_metric>
user_sum_dict = user_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_count_dict = user_agg['count'].astype('int16' ).to_dict(defaultdict(int))
val_loss, val_acc = model2.evaluate(X, Y)
 val_acc<save_to_csv>
del user_agg
gc.collect()
task_container_sum_dict = task_container_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
task_container_count_dict = task_container_agg['count'].astype('int32' ).to_dict(defaultdict(int))
task_container_std_dict = task_container_agg['var'].astype('float16' ).to_dict(defaultdict(int))
explanation_sum_dict = explanation_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
explanation_count_dict = explanation_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del task_container_agg
del explanation_agg
gc.collect()
results = model2.predict(test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label")
 submission3 = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission3.to_csv("sub3.csv",index=False )<load_from_csv>
user_lecture_sum_dict = user_lecture_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_lecture_count_dict = user_lecture_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del user_lecture_agg
gc.collect()
train = pd.read_csv('.. /input/digit-recognizer/train.csv')
 test_data = pd.read_csv('.. /input/digit-recognizer/test.csv')
 print(train.info() )<prepare_x_and_y>
max_timestamp_u_dict=max_timestamp_u.set_index('user_id' ).to_dict()
max_timestamp_u_dict2=max_timestamp_u2.set_index('user_id' ).to_dict()
max_timestamp_u_dict3=max_timestamp_u3.set_index('user_id' ).to_dict()
user_prior_question_elapsed_time_dict=user_prior_question_elapsed_time.set_index('user_id' ).to_dict()
del max_timestamp_u
del max_timestamp_u2
del max_timestamp_u3
del user_prior_question_elapsed_time
gc.collect()
X_train = train.iloc[:,1:]
y_train = train.iloc[:,0]<prepare_x_and_y>
attempt_no_sum_dict = attempt_no_agg['sum'].to_dict(defaultdict(int))
del attempt_no_agg
gc.collect()
X_train = X_train/255.0
test_data = test_data/255.0
X_train = X_train.values.reshape(-1,28,28,1)
 test_data = test_data.values.reshape(-1,28,28,1 )<import_modules>
def get_max_attempt(user_id,content_id):
k =(user_id,content_id)
 if k in attempt_no_sum_dict.keys() :
attempt_no_sum_dict[k]+=1
return attempt_no_sum_dict[k]
attempt_no_sum_dict[k] = 1
return attempt_no_sum_dict[k]
import tensorflow as tf
from tensorflow import keras
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D, Dropout, Lambda, Flatten, Dense
from keras.utils.np_utils import to_categorical<categorify>
y_train = to_categorical(y_train,num_classes=10 )<split>
iter_test = env.iter_test()
prior_test_df = None
prev_test_df = None
X_train, X_test, y_train, y_test = train_test_split(X_train, y_train, test_size=0.1, random_state=42 )<choose_model_class>
N=[0.4,0.6]
model = Sequential()
model.add(Conv2D(32,(5,5),padding='Same', activation='relu', kernel_initializer='glorot_uniform', input_shape=(28, 28, 1)))
 model.add(Conv2D(32,(5,5),padding='Same', activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
model.add(Conv2D(64,(3,3), padding='Same', activation='relu'))
model.add(Conv2D(64,(3,3), padding='Same', activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(10, activation='softmax'))<choose_model_class>
%%time
for(test_df, sample_prediction_df)in iter_test:
test_df1=test_df.copy()
if(prev_test_df is not None)&(psutil.virtual_memory().percent<90):
print(psutil.virtual_memory().percent)
 prev_test_df['answered_correctly'] = eval(test_df1['prior_group_answers_correct'].iloc[0])
 prev_test_df = prev_test_df[prev_test_df.content_type_id == False]
prev_group = prev_test_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
prev_group_content = prev_group[prev_user_id][0]
prev_group_answered_correctly = prev_group[prev_user_id][1]
if prev_user_id in group.index:
group[prev_user_id] =(np.append(group[prev_user_id][0], prev_group_content),
np.append(group[prev_user_id][1], prev_group_answered_correctly))
else:
group[prev_user_id] =(prev_group_content, prev_group_answered_correctly)
 if len(group[prev_user_id][0])> MAX_SEQ:
new_group_content = group[prev_user_id][0][-MAX_SEQ:]
new_group_answered_correctly = group[prev_user_id][1][-MAX_SEQ:]
group[prev_user_id] =(new_group_content, new_group_answered_correctly)
 prev_test_df = test_df1.copy()
test_df1 = test_df1[test_df1.content_type_id == False]
test_dataset = TestDataset(group, test_df1, n_skill, max_seq=MAX_SEQ)
 test_dataloader = DataLoader(test_dataset, batch_size=len(test_df1), shuffle=False)
 outs = []
test_df1 = test_df1[test_df1.content_type_id == False]
test_dataset = TestDataset(group, test_df1, n_skill, max_seq=MAX_SEQ)
 test_dataloader = DataLoader(test_dataset, batch_size=len(test_df1), shuffle=False)
 item = next(iter(test_dataloader))
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, _ = nn_model(x, target_id)
 output = torch.sigmoid(output)
 output = output[:, -1]
outs = output.cpu().numpy()
if prior_test_df is not None:
prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)
 prior_test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 prior_test_df.prior_question_had_explanation=prior_test_df.prior_question_had_explanation.astype('int8')
 user_ids = prior_test_df['user_id'].values
targets = prior_test_df[target].values
for user_id, answered_correctly in zip(user_ids,targets):
user_sum_dict[user_id] += answered_correctly
user_count_dict[user_id] += 1
prior_test_df = test_df.copy()
question_len=len(test_df[test_df['content_type_id'] == 0])
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df.prior_question_had_explanation=test_df.prior_question_had_explanation.astype('int8')
 test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)
 user_lecture_sum = np.zeros(question_len, dtype=np.int16)
 user_lecture_count = np.zeros(question_len, dtype=np.int16)
 user_sum = np.zeros(question_len, dtype=np.int16)
 user_count = np.zeros(question_len, dtype=np.int16)
 task_container_sum = np.zeros(question_len, dtype=np.int32)
 task_container_count = np.zeros(question_len, dtype=np.int32)
 task_container_std = np.zeros(question_len, dtype=np.float16)
 explanation_sum = np.zeros(question_len, dtype=np.int32)
 explanation_count = np.zeros(question_len, dtype=np.int32)
 delta_prior_question_elapsed_time = np.zeros(question_len, dtype=np.int32)
 attempt_no_count = np.zeros(question_len, dtype=np.int16)
 lagtime = np.zeros(question_len, dtype=np.float32)
 lagtime2 = np.zeros(question_len, dtype=np.float32)
 lagtime3 = np.zeros(question_len, dtype=np.float32)
 i=0
for j,(user_id,prior_question_had_explanation,content_type_id,prior_question_elapsed_time,timestamp, content_id,task_container_id)in enumerate(zip(test_df['user_id'].values,test_df['prior_question_had_explanation'].values,test_df['content_type_id'].values,test_df['prior_question_elapsed_time'].values,test_df['timestamp'].values, test_df['content_id'].values, test_df['task_container_id'].values)) :
user_lecture_sum_dict[user_id] += content_type_id
user_lecture_count_dict[user_id] += 1
if(content_type_id==1):
x=1
if(content_type_id==0):
user_lecture_sum[i] = user_lecture_sum_dict[user_id]
user_lecture_count[i] = user_lecture_count_dict[user_id]
user_sum[i] = user_sum_dict[user_id]
user_count[i] = user_count_dict[user_id]
task_container_sum[i] = task_container_sum_dict[task_container_id]
task_container_count[i] = task_container_count_dict[task_container_id]
task_container_std[i]=task_container_std_dict[task_container_id]
explanation_sum_dict[user_id] += prior_question_had_explanation
explanation_count_dict[user_id] += 1
explanation_sum[i] = explanation_sum_dict[user_id]
explanation_count[i] = explanation_count_dict[user_id]
if user_id in max_timestamp_u_dict['max_time_stamp'].keys() :
lagtime[i]=timestamp-max_timestamp_u_dict['max_time_stamp'][user_id]
if(max_timestamp_u_dict2['max_time_stamp2'][user_id]==lagtime_mean2):
lagtime2[i]=lagtime_mean2
lagtime3[i]=lagtime_mean3
else:
lagtime2[i]=timestamp-max_timestamp_u_dict2['max_time_stamp2'][user_id]
if(max_timestamp_u_dict3['max_time_stamp3'][user_id]==lagtime_mean3):
lagtime3[i]=lagtime_mean3
else:
lagtime3[i]=timestamp-max_timestamp_u_dict3['max_time_stamp3'][user_id]
max_timestamp_u_dict3['max_time_stamp3'][user_id]=max_timestamp_u_dict2['max_time_stamp2'][user_id]
max_timestamp_u_dict2['max_time_stamp2'][user_id]=max_timestamp_u_dict['max_time_stamp'][user_id]
max_timestamp_u_dict['max_time_stamp'][user_id]=timestamp
else:
lagtime[i]=lagtime_mean
max_timestamp_u_dict['max_time_stamp'].update({user_id:timestamp})
 lagtime2[i]=lagtime_mean2
max_timestamp_u_dict2['max_time_stamp2'].update({user_id:lagtime_mean2})
 lagtime3[i]=lagtime_mean3
max_timestamp_u_dict3['max_time_stamp3'].update({user_id:lagtime_mean3})
 if user_id in user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].keys() :
delta_prior_question_elapsed_time[i]=prior_question_elapsed_time-user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]=prior_question_elapsed_time
else:
delta_prior_question_elapsed_time[i]=delta_prior_question_elapsed_time_mean
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].update({user_id:prior_question_elapsed_time})
 i=i+1
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df=test_df.merge(questions_df.loc[questions_df.index.isin(test_df['content_id'])],
how='left', on='content_id', right_index=True)
 test_df['user_lecture_lv'] = user_lecture_sum / user_lecture_count
test_df['user_lecture_sum'] = user_lecture_sum
test_df['user_interaction_count'] = user_lecture_count
test_df['user_interaction_timestamp_mean'] = test_df['timestamp']/user_lecture_count
test_df['user_correctness'] = user_sum / user_count
test_df['user_uncorrect_count'] =user_count-user_sum
test_df['user_correct_count'] =user_sum
test_df['task_container_correctness'] = task_container_sum / task_container_count
test_df['task_container_cor_count'] = task_container_sum
test_df['task_container_uncor_count'] =task_container_count-task_container_sum
test_df['task_container_std'] = task_container_std
test_df['explanation_mean'] = explanation_sum / explanation_count
test_df['explanation_true_count'] = explanation_sum
test_df['explanation_false_count'] = explanation_count-explanation_sum
test_df['delta_prior_question_elapsed_time'] = delta_prior_question_elapsed_time
test_df["attempt_no"] = test_df[["user_id", "content_id"]].apply(lambda row: get_max_attempt(row["user_id"], row["content_id"]), axis=1)
 test_df["lagtime"]=lagtime
test_df["lagtime2"]=lagtime2
test_df["lagtime3"]=lagtime3
test_df['timestamp']=test_df['timestamp']/(1000*3600)
 test_df.timestamp=test_df.timestamp.astype('float16')
 test_df['lagtime']=test_df['lagtime']/(1000*3600)
 test_df.lagtime=test_df.lagtime.astype('float32')
 test_df['lagtime2']=test_df['lagtime2']/(1000*3600)
 test_df.lagtime2=test_df.lagtime2.astype('float32')
 test_df['lagtime3']=test_df['lagtime3']/(1000*3600)
 test_df.lagtime3=test_df.lagtime3.astype('float32')
 test_df['user_interaction_timestamp_mean']=test_df['user_interaction_timestamp_mean']/(1000*3600)
 test_df.user_interaction_timestamp_mean=test_df.user_interaction_timestamp_mean.astype('float32')
 test_df['user_correctness'].fillna(0.67, inplace=True)
 sub_preds = np.zeros(test_df.shape[0])
 for i, model in enumerate(clfs, 1):
test_preds = model.predict(test_df[features])
 sub_preds += test_preds
o2=sub_preds / len(clfs)
 test_df[target]=0.5*np.array(outs)+0.5*np.array(o2)
 env.predict(test_df[['row_id', target]] )