Kaggle competition: Riiid Answer Correctness Prediction
def get_model(shape, n_class):
model = Sequential()
model.add(Conv2D(64,(4,4), input_shape=shape))
model.add(MaxPool2D(( 2,2)))
 model.add(BatchNormalization())
 model.add(tf.keras.layers.Activation("selu")) ,
model.add(Conv2D(128,(3,3)))
 model.add(MaxPool2D(( 2,2)))
 model.add(BatchNormalization())
 model.add(tf.keras.layers.Activation("selu")) ,
model.add(Conv2D(256,(2,2)))
 model.add(MaxPool2D(( 2,2)))
 model.add(Dropout(0.2))
model.add(BatchNormalization())
 model.add(Flatten())
 model.add(tf.keras.layers.Activation("selu"))
model.add(Dense(64, kernel_initializer="he_normal", use_bias=False))
model.add(Dropout(0.2))
model.add(BatchNormalization())
 model.add(tf.keras.layers.Activation("selu"))
model.add(Dense(32, kernel_initializer="he_normal", use_bias=False))
model.add(BatchNormalization())
 model.add(Dense(n_class, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model 
<choose_model_class>
def reduce_mem_usage(df, use_float16=False,verbose=True):

 start_mem = df.memory_usage().sum() / 1024**2
if verbose :print("Memory usage of dataframe is {:.2f} MB".format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == "int":
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype("category")
 end_mem = df.memory_usage().sum() / 1024**2
if verbose:print("Memory usage after optimization is: {:.2f} MB".format(end_mem))
if verbose:print("Decreased by {:.1f}%".format(100 *(start_mem - end_mem)/ start_mem))
return df
def get_adv_model(shape, n_class):
model = Sequential([
Conv2D(64,(4,4), input_shape=shape, dynamic=True),
MaxPool2D(( 2,2)) ,
BatchNormalization() ,
tf.keras.layers.Activation("selu"),
Conv2D(128,(3,3)) ,
MaxPool2D(( 2,2)) ,
BatchNormalization() ,
tf.keras.layers.Activation("selu"),
Conv2D(256,(2,2)) ,
MaxPool2D(( 2,2)) ,
Dropout(0.2),
BatchNormalization() ,
Flatten() ,
tf.keras.layers.Activation("selu"),
Dense(64, kernel_initializer="he_normal", use_bias=False),
Dropout(0.2),
BatchNormalization() ,
tf.keras.layers.Activation("selu"),
Dense(32, kernel_initializer="he_normal", use_bias=False),
BatchNormalization() ,
Dense(n_class, activation='softmax')])
 adv_config = nsl.configs.make_adv_reg_config(multiplier=0.2, adv_step_size=0.05)
 adv_model = nsl.keras.AdversarialRegularization(model, adv_config=adv_config)
 adv_model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'], run_eagerly=True)
 return adv_model 
<choose_model_class>
import random
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import gc
from sklearn.metrics import roc_auc_score
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import StandardScaler,MinMaxScaler,PolynomialFeatures
import json
from sklearn.metrics import roc_auc_score,accuracy_score
import lightgbm as lgb
import dill
import time
from sklearn.pipeline import Pipeline,FeatureUnion
from sklearn.decomposition import PCA
from sklearn.feature_selection import SelectKBest, chi2, f_regression
from sklearn.model_selection import train_test_split
from sklearn.base import BaseEstimator, ClassifierMixin
import xgboost as xgb
import tensorflow as tf
from tensorflow import keras
def get_sample_model(shape, n_class):
model = tf.keras.Sequential([
tf.keras.Input(shape, name='feature'),
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(128, activation=tf.nn.relu),
tf.keras.layers.Dense(n_class, activation=tf.nn.softmax)
 ])
 adv_config = nsl.configs.make_adv_reg_config(multiplier=0.2, adv_step_size=0.05)
 adv_model = nsl.keras.AdversarialRegularization(model, adv_config=adv_config)
 adv_model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
 return adv_model<define_variables>
np.random.seed(59)
 tf.random.set_seed(59)
 random.seed(59)
 path_train1 = '.. /input/riiid-preprocess-and-balance-the-dataset/train0.feather'
path_train2 = '.. /input/riiid-preprocess-and-balance-the-dataset/train1.feather'
path_content = '.. /input/riiid-preprocess-and-balance-the-dataset/content.feather'
path_metadata = '.. /input/riiid-preprocess-and-balance-the-dataset/metadata.dill'
features = get_features(train)
 labels = get_labels(train)
<choose_model_class>
SUBMIT = True
model = get_adv_model(x[0].shape, 10)
 print(model )<split>
%%skip SUBMIT
metadata = dill.load(open(path_metadata,'rb'))
target = 'answered_correctly'
train_df = pd.read_feather(path_train1)
 train_df = train_df.append(pd.read_feather(path_train2))
train_df.shape
train_x = x[:36000]
test_x = x[36000:]
labels_train = labels[:36000]
labels_test = labels[36000:]<init_hyperparams>
%%skip SUBMIT
for key in metadata:
if key in [('content_id',),('bundle_id',)]:
print(key)
 train_df = train_df.merge(metadata[key],how='left',on=key)
 del metadata
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(train_x )<choose_model_class>
def add_mean(df,remove=False):
for col in ['user_id','content_id','bundle_id',]:
c1 = f'{col}_roll_count'
c2 = f'{col}_roll_sum'
c3 = f'{col}_roll_mean'
df[c3] =(df[c2]/df[c1] ).astype(np.float32)
 if remove:
del df[c1],df[c2]
for col in ['bundle','content']:
c1 = f'{col}_count'
c2 = f'{col}_sum'
c3 = f'{col}_mean'
df[c3] =(df[c2]/df[c1] ).astype(np.float32)
 if remove:
del df[c1],df[c2]
df.fillna(-1,inplace=True)
 return df
earlyStop = tf.keras.callbacks.EarlyStopping(monitor='val_acc', patience=10)
 reduceLROnPlateau = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )<train_model>
%%skip SUBMIT
ignore_columns = ['user_answer','user_id','row_id']
train_df = add_mean(train_df,remove=False )
model.fit(datagen.flow(train_x, labels_train, batch_size=32),
steps_per_epoch=len(train_x)// 32,
epochs=5)
<prepare_output>
%%skip SUBMIT
features = list(train_df.columns)
 for col in ignore_columns+['answered_correctly']:
if col in features:features.remove(col)
 print(features)
 print('Features size: ',len(features))
dill.dump(features,open('features.dill','wb'))
dill.dump(ignore_columns,open('ignore_columns.dill','wb'))
model.evaluate({'feature': test_x, 'label': np.array(labels_test)} )<load_from_csv>
%%skip SUBMIT
X_test,y_test = None,None
if True:
train_df,test_df = train_test_split(train_df,
stratify=train_df[target],
test_size=0.2,
random_state=1)
 test_df.drop(columns=ignore_columns,inplace=True)
 X_test,y_test = test_df[features].astype(np.float32),test_df[target]
del test_df
train_df.drop(columns=ignore_columns,inplace=True)
 X,y = train_df[features].astype(np.float32),train_df[target]
del train_df
X.shape
test = pd.read_csv('/kaggle/input/test.csv')
 test.head()<prepare_x_and_y>
gc.collect()
test_normalized = test/ 255
x_test = np.array(test_normalized ).reshape(-1,28,28,1 )<predict_on_test>
class CustomLGBClassifier(BaseEstimator, ClassifierMixin):
def __init__(self):
self.params = {
'objective': 'binary',
'learning_rate': 0.1,
'verbose': 1,
'device': gpu,
'metrics': ['binary','auc'],
'nthread':4,
'seed':1,
}
self.model = None
self.pipe = Pipeline([ 
('scaler',MinMaxScaler()),
])
 self.score_value = None
def fit(self, X,y, X_test=None,y_test=None):
X = self.pipe.fit_transform(X,y)
 train_data = lgb.Dataset(X, label=y)
 valid_sets = [train_data,]
val_data = None
if X_test is not None:
X_test = self.pipe.transform(X_test)
 val_data = lgb.Dataset(X_test, label=y_test)
 valid_sets.append(val_data)
 self.model = lgb.train(self.params,
train_data,
num_boost_round=3000,
valid_sets=valid_sets,
early_stopping_rounds=50,
verbose_eval=True 
)
 return self
def predict(self, X):
return self.model.predict(self.pipe.transform(X))
def predict_proba(self,X):
return self.predict(X)[:,np.newaxis]
def score(self,X,y):
y_pred = self.predict(X)
 auc = round(roc_auc_score(y, y_pred),2)
 acc = round(accuracy_score(y, y_pred>0.5),2)
 return auc,acc
lo = [0] * len(x_test)
 predictions = model.predict({'feature':x_test, 'label':np.array(lo)} )<save_to_csv>
train_lgbc = False
response = []
for i, prediction in enumerate(predictions):
response.append([i + 1, np.argmax(prediction)])
 resposta = pd.DataFrame(response, columns=['ImageId', 'Label'])
 resposta.to_csv('output.csv', index=False)
<install_modules>
%%skip SUBMIT or not train_lgbc
model = CustomLGBClassifier()
model.fit(X,y,X_test,y_test)
 dill.dump(model,open(f'lgb_model.dill','wb'))
!pip install seaborn==0.11.0<set_options>
class CustomXGBClassifier(BaseEstimator, ClassifierMixin):
def __init__(self,):
self.params = {
'objective': 'binary:logistic',
'learning_rate': 0.1,
"max_depth": 10,
'eval_metric':'auc',
'subsample':0.6,
'colsample_bytree':0.6,
'tree_method': 'gpu_hist' if gpu else None,
'seed': 1
}
self.model = None
self.pipe = Pipeline([ 
('scaler',MinMaxScaler()),
])
 self.score_value = None
self.best_ntree_limit = 0
def fit(self, X,y, X_test=None,y_test=None):
X = self.pipe.fit_transform(X,y)
 train_data = xgb.DMatrix(X, label=y,feature_names=features)
 evals = [(train_data, 'train'),]
if X_test is not None:
X_test = self.pipe.transform(X_test)
 val_data = xgb.DMatrix(X_test, label=y_test,feature_names=features)
 evals.append(( val_data, 'eval'))
self.model = xgb.train(self.params,train_data,3000,
evals = evals,
early_stopping_rounds=50,
verbose_eval=True 
)
 self.best_ntree_limit = self.model.best_ntree_limit
return self
def plot_features(self):
xgb.plot_importance(self.model,show_values=True,max_num_features=len(features))
def init_thread(self):
self.best_ntree_limit = self.model.best_ntree_limit
self.model.save_model('tmp.model')
 bst = xgb.Booster({'nthread': 4})
 bst.load_model('tmp.model')
 self.model = bst
def predict(self, X):
return self.model.predict(
 xgb.DMatrix(self.pipe.transform(X),feature_names=features), 
)
 def predict_proba(self,X):
return self.predict(X)[:,np.newaxis]
def score(self,X,y):
y_pred = self.predict(X)
 auc = round(roc_auc_score(y, y_pred),2)
 return auc
pd.set_option('display.max_colwidth', None)
 pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', None)
 print(sns.__version__)
 sns.set(rc={"font.size":18,"axes.titlesize":30,"axes.labelsize":18,
"axes.titlepad":22, "axes.labelpad":18, "legend.fontsize":15,
"legend.title_fontsize":15, "figure.titlesize":35})
 init_notebook_mode(connected=True)
<load_from_csv>
train_xgbc = False
df_train = pd.read_csv("/kaggle/input/titanic/train.csv")
 df_test = pd.read_csv("/kaggle/input/titanic/test.csv")
 df_train.head(5 )<data_type_conversions>
%%skip SUBMIT or not train_xgbc
model = CustomXGBClassifier()
model.fit(X,y,X_test,y_test)
 print('Saving...')
 dill.dump(model,open(f'xgb_model.dill','wb'))
print('Done...' )
for col in ['Sex', 'Cabin', 'Ticket', 'Embarked']:
df_train[col] = df_train[col].astype('category')
 df_test[col] = df_test[col].astype('category')
 df_train.info(verbose=True )<concatenate>
train_nn = False
full_df = pd.concat([df_train, df_test] ).reset_index(drop=True)
 train_shape = df_train.shape
test_shape = df_test.shape<data_type_conversions>
%%skip SUBMIT or not train_nn
model_path="best_nn_model.h5"
callbacks_list = [
keras.callbacks.ModelCheckpoint(model_path,
verbose=0,
monitor='auc',
save_best_only=True,
mode='max'),
]
batch_size = 2**14
val_data = None if X_test is None else(X_test,y_test )
full_df.loc[:, 'Sex'] =(full_df.loc[:, 'Sex'] == 'female' ).astype(int )<categorify>
%%skip SUBMIT or not train_nn
layers = [keras.layers.BatchNormalization(input_shape=(len(features),)) ,]
for u in range(5):
layers.extend([
keras.layers.BatchNormalization(input_shape=(len(features),)) ,
keras.layers.Dense(256),
keras.layers.BatchNormalization() ,
keras.layers.Activation('relu'),
])
 model = keras.Sequential([
*layers,
keras.layers.Dense(1,activation='sigmoid'),
])
 if os.path.exists("nn_model.h5"):
print('Loading...')
 model = keras.models.load_model("nn_model.h5")
 model.compile(loss=keras.losses.binary_crossentropy,
optimizer=keras.optimizers.Adam(0.01),
metrics=[
'acc',
keras.metrics.AUC(name='auc'),
] 
)
 model.summary()
full_df['Title_C'] = full_df['Title']
full_df = pd.get_dummies(full_df, columns=["Embarked","Title_C"],\
prefix=["Emb","Title"], drop_first=False)
 title_dict = {'Mr': 1, 'Miss': 2, 'Mrs': 3, 'Other': 4, 'Royal': 5, 'Master': 6}
full_df['Title'] = full_df['Title'].map(title_dict ).astype('int' )<data_type_conversions>
%%skip SUBMIT or not train_nn
history = model.fit(X,y,
batch_size=batch_size,
validation_data=val_data,
epochs=100,
callbacks=callbacks_list, 
)
 model.save("nn_model.h5" )
Fsize_dict = {'Alone':3, 'Small':2, 'Big':1}
full_df['Fsize_Cat'] = full_df['Fsize_Cat'].map(Fsize_dict ).astype('int' )<feature_engineering>
%%skip not SUBMIT
lock = mp.Lock()
full_df['Family_Name'] = full_df['Name'].str.extract('([A-Za-z]+.[A-Za-z]+)\,', expand=True )<feature_engineering>
use_lgb = True
use_xgb = False
use_nn = False
MEAN_SURVIVAL_RATE = round(np.mean(df_train['Survived']), 4)
 full_df['Family_Friends_Surv_Rate'] = MEAN_SURVIVAL_RATE
full_df['Surv_Rate_Invalid'] = 1
for _, grp_df in full_df[['Survived', 'Family_Name', 'Fare', 'Ticket', 'PassengerId']].groupby(['Family_Name', 'Fare']):
if(len(grp_df)> 1):
if(grp_df['Survived'].isnull().sum() != len(grp_df)) :
for ind, row in grp_df.iterrows() :
full_df.loc[full_df['PassengerId'] == row['PassengerId'],
'Family_Friends_Surv_Rate'] = round(grp_df['Survived'].mean() , 4)
 full_df.loc[full_df['PassengerId'] == row['PassengerId'],
'Surv_Rate_Invalid'] = 0
for _, grp_df in full_df[['Survived', 'Family_Name', 'Fare', 'Ticket', 'PassengerId', 'Family_Friends_Surv_Rate']].groupby('Ticket'):
if(len(grp_df)> 1):
for ind, row in grp_df.iterrows() :
if(row['Family_Friends_Surv_Rate'] == 0.) |(row['Family_Friends_Surv_Rate'] == MEAN_SURVIVAL_RATE):
if(grp_df['Survived'].isnull().sum() != len(grp_df)) :
full_df.loc[full_df['PassengerId'] == row['PassengerId'],
'Family_Friends_Surv_Rate'] = round(grp_df['Survived'].mean() , 4)
 full_df.loc[full_df['PassengerId'] == row['PassengerId'],
'Surv_Rate_Invalid'] = 0<categorify>
%%skip not(use_lgb and SUBMIT)
 model = dill.load(open('lgb_model.dill','rb'))
model
full_df['Cabin'] = full_df['Cabin'].astype('category')
 full_df['Cabin'] = full_df['Cabin'].cat.add_categories('U')
 full_df['Cabin_Clean'] = full_df['Cabin'].fillna('U')
 full_df['Cabin_Clean'] = full_df['Cabin_Clean'].str.strip(' ' ).str[0]
cabin_dict = {'A':9, 'B':8, 'C':7, 'D':6, 'E':5, 'F':4, 'G':3, 'T':2, 'U':1}
full_df['Cabin_Clean'] = full_df['Cabin_Clean'].map(cabin_dict ).astype('int')
<categorify>
%%skip not(use_xgb and SUBMIT)
 model = dill.load(open('xgb_model.dill','rb'))
model
full_df = pd.get_dummies(full_df, columns=["Tkt_Clean"],\
prefix=["Tkt"], drop_first=True )<data_type_conversions>
%%skip not(use_nn and SUBMIT)
 model = keras.models.load_model("best_nn_model.h5" )
def fare_cat(fare):
if fare <= 7.0:
return 1
elif fare <= 39 and fare > 7.0:
return 2
else:
return 3
full_df.loc[:, 'Fare_Cat'] = full_df['Fare'].apply(fare_cat ).astype('int' )<feature_engineering>
%%skip not SUBMIT
metadata_info = dill.load(open('/kaggle/input/riiid-metadata-to-sqlite/metadata_info.dill','rb'))
features = dill.load(open('features.dill','rb'))
ignore_columns = dill.load(open('ignore_columns.dill','rb'))
content_df = pd.read_feather('.. /input/riiid-preprocess-and-balance-the-dataset/content.feather')
 print('Features size: ',len(features))
full_df.loc[:, 'Fare_Family_Size'] = full_df['Fare']/full_df['Family_Size']
full_df.loc[:, 'Fare_Cat_Pclass'] = full_df['Fare_Cat']*full_df['Pclass']
full_df.loc[:, 'Fare_Cat_Title'] = full_df['Fare_Cat']*full_df['Title']
full_df.loc[:, 'Fsize_Cat_Title'] = full_df['Fsize_Cat']*full_df['Title']
full_df.loc[:, 'Fsize_Cat_Fare_Cat'] = full_df['Fare_Cat']/full_df['Fsize_Cat'].astype('int')
 full_df.loc[:, 'Pclass_Title'] = full_df['Pclass']*full_df['Title']
full_df.loc[:, 'Fsize_Cat_Pclass'] = full_df['Fsize_Cat']*full_df['Pclass']<define_variables>
%%skip not SUBMIT
for key in metadata_info:
name = f"db.{'_'.join(key)}.sqlite"
engine = create_engine(f'sqlite:///{name}?check_same_thread=False', echo=False)
 sqlite_connection = engine.connect()
cols = metadata_info[key][-1]
cols = [column(col)for col in cols]+[column('index'),]
tab = table('_'.join(key), *cols)
 metadata_info[key][0] =(sqlite_connection,tab )
colsToRemove = []
cols = ['Tkt_AQ', 'Tkt_AS', 'Tkt_C', 'Tkt_CA',
'Tkt_CASOTON', 'Tkt_FC', 'Tkt_FCC', 'Tkt_Fa', 'Tkt_LINE', 'Tkt_LP',
'Tkt_NUM', 'Tkt_PC', 'Tkt_PP', 'Tkt_PPP', 'Tkt_SC', 'Tkt_SCA',
'Tkt_SCAH', 'Tkt_SCAHBasle', 'Tkt_SCOW', 'Tkt_SCPARIS', 'Tkt_SCParis',
'Tkt_SOC', 'Tkt_SOP', 'Tkt_SOPP', 'Tkt_SOTONO', 'Tkt_SOTONOQ',
'Tkt_SP', 'Tkt_STONO', 'Tkt_STONOQ', 'Tkt_SWPP', 'Tkt_WC',
'Tkt_WEP', 'Fare_Cat', 'Fare_Family_Size', 'Fare_Cat_Pclass',
'Fare_Cat_Title', 'Fsize_Cat_Title', 'Fsize_Cat_Fare_Cat',
'Pclass_Title', 'Fsize_Cat_Pclass']
for col in cols:
if full_df[col][:train_shape[0]].std() == 0:
colsToRemove.append(col)
 full_df.drop(colsToRemove, axis=1, inplace=True)
 print("Removed `{}` Constant Columns
".format(len(colsToRemove)))
 print(colsToRemove )<define_variables>
def build_index(x):
return '_'.join(map(lambda x:str(int(x)) ,x)) if type(x)not in [int,float] else str(int(x))
def getValues(args):
key,df,cache= args
df = df[list(key)]
index = df.index
df = df[list(key)]
db,tab = metadata_info[key][0]
cols = metadata_info[key][-1]
ids = df.apply(build_index,axis=1 ).values
query = select([tab.c['index'],] + [tab.c[col] for col in cols] ).where(tab.c.index.in_(np.unique(ids)))
 result = list(db.execute(query))
res_df = pd.DataFrame(result,columns=['tmp_key']+cols)
 res_df.set_index('tmp_key',inplace=True)
 df['tmp_key'] = ids
df = df.merge(res_df,how='left',on='tmp_key' ).fillna(0)
 cache[key] = df.groupby([*key,] ).tail(1 ).set_index([*key,])
 df = df.drop(columns=[*key,'tmp_key'])
 df.index = index
return df
def merge_metadata(df):
cache = {}
workers= []
queue = Queue()
for key in metadata_info:
worker = Worker(getValues,(key,df,cache),queue)
 worker.start()
workers.append(worker)
 frames = [queue.get() for _ in workers]
frames.append(df)
 for worker in workers:
worker.join()
return pd.concat(frames,axis=1),cache
def update_metadata(args):
df,key = args
if not df.shape[0]:
return
key =(key,)if type(key)== str else tuple(key)
 if key not in metadata_info:
return 
(db,tab),cols = metadata_info[key]
cols = [c for c in cols if c not in ignore_columns]
if all(k in df.index.names for k in key):
df = df.loc[:,[c for c in cols if c not in key]]
else:
df = df.set_index(list(key)).loc[:,cols]
db_index = list(map(build_index,df.index))
df.index = db_index
query = delete(tab ).where(tab.c.index.in_(db_index))
db.execute(query)
 df.to_sql('_'.join(key), db, if_exists='append')
 def add_previous_state(df,key,cache):
key =(key,)if type(key)== str else tuple(key)
 if not all(k in df.index.names for k in key):
df.set_index(list(key),inplace=True)
 index = df.index
cols = [c for c in df.columns if c not in ignore_columns]
if not cols:
return df
return(df+cache[key])[cols]
imp_features = ['Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'Title',
'Name_Length', 'Emb_C', 'Emb_Q', 'Emb_S','Family_Size',
'Fsize_Cat', 'Family_Friends_Surv_Rate', 'Surv_Rate_Invalid',
'Cabin_Clean','Ticket_Frequency', 'Tkt_AS', 'Tkt_C', 'Tkt_CA',
'Tkt_CASOTON', 'Tkt_FC', 'Tkt_FCC', 'Tkt_Fa', 'Tkt_LINE',
'Tkt_NUM', 'Tkt_PC', 'Tkt_PP', 'Tkt_PPP', 'Tkt_SC', 'Tkt_SCA',
'Tkt_SCAH', 'Tkt_SCAHBasle', 'Tkt_SCOW', 'Tkt_SCPARIS', 'Tkt_SCParis',
'Tkt_SOC', 'Tkt_SOP', 'Tkt_SOPP', 'Tkt_SOTONO', 'Tkt_SOTONOQ',
'Tkt_SP', 'Tkt_STONO', 'Tkt_SWPP', 'Tkt_WC',
'Tkt_WEP', 'Fare_Cat', 'Fare_Family_Size', 'Fare_Cat_Pclass',
'Fare_Cat_Title', 'Fsize_Cat_Title', 'Fsize_Cat_Fare_Cat',
'Pclass_Title', 'Fsize_Cat_Pclass']
imputer = KNNImputer(n_neighbors=10, missing_values=np.nan)
 imputer.fit(full_df[imp_features] )<create_dataframe>
def user_roll_features(df,cols):
metadata = {}
for col in cols:
c1 = f'{col}_roll_count'
c2 = f'{col}_roll_sum'
df_tmp = df[[col,'user_id','answered_correctly',]]
grp = df_tmp.groupby([col,'user_id'])
 df_tmp[c1] = grp.answered_correctly.cumcount() +1
df_tmp[c2] = grp.answered_correctly.cumsum()
wincols = []
for win in [50,10]:
sum_col = f'{col}_win{win}_sum'
count_col = f'{col}_win{win}_count'
df_tmp[sum_col] = df_tmp[c2]
df_tmp[count_col] = df_tmp[c1]
wincols.append(sum_col)
 wincols.append(count_col)
 grp = df_tmp.groupby(['user_id',col] ).tail(1)
 metadata[('user_id',col)] = grp.set_index(['user_id',col])[[c1,c2,*wincols]]
return metadata
full_df.loc[:, imp_features] = pd.DataFrame(imputer.transform(full_df[imp_features]), index=full_df.index, columns = imp_features )<split>
skip_test = True
df_train_final = full_df[:train_shape[0]]
df_test_final = full_df[train_shape[0]:]<drop_column>
%%skip skip_test
val_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',
header=0,
skiprows=range(1,1000000*5),
nrows=500000)
 val_df['row_id'] = range(val_df.shape[0])
 val_df = val_df[['row_id', 'timestamp', 'user_id', 'content_id', 'content_type_id',
'task_container_id', 'prior_question_elapsed_time',
'prior_question_had_explanation',
'answered_correctly','user_answer']]
viz_features = ['PassengerId', 'Survived', 'Pclass', 'Name', 'Sex',
'Age', 'SibSp','Parch', 'Ticket', 'Fare', 'Cabin',
'Title', 'Name_Length', 'Family_Friends_Surv_Rate',
'Ticket_Frequency']
train_viz = df_train_final[viz_features]<categorify>
%%skip skip_test
prev_test_df = None
prev_cache = None
auc_sum = 0
acc_sum = 0
df = val_df
count = 0
while df.shape[0]:
count+=1
selector = df.task_container_id == df.task_container_id.min()
test_df = df.loc[selector]
df = df.loc[~selector]
print(f'{test_df.shape[0]}/{df.shape[0]}')
 test_df = test_df.merge(content_df,how='left',on=['content_id',])
 test_df.prior_question_had_explanation = test_df.prior_question_had_explanation.fillna(False ).astype(np.int8)
 test_state = test_df.loc[test_df['content_type_id'] == 0]
test_state.replace([np.inf, -np.inf], 999999999,inplace=True)
 test_state,cache = merge_metadata(test_state)
 test_state['prior_duration'] = test_state.timestamp-test_state.prev_timestamp
test_state = add_mean(test_state)
 test_state['answered_correctly'] = model.predict(test_state[features])
 y = test_df[test_df['content_type_id'] == 0]['answered_correctly']
y_pred = test_state['answered_correctly']
auc = roc_auc_score(y, y_pred)
 acc = accuracy_score(y, y_pred>0.5)
 auc_sum += auc
acc_sum += acc
print(f"AUC:{round(auc,4)} | Accuracy: {round(acc,4)} | AUC MEAN: {round(auc_sum/count,4)} | ACC MEAN: {round(acc_sum/count,4)}")
 if prev_test_df is not None:
workers = []
prev_test_df = prev_test_df[prev_test_df.content_type_id == 0]
user_df = prev_test_df[['user_id','answered_correctly']]
grp = user_df.groupby(['user_id'])
 user_df['user_id_roll_sum'] = grp['answered_correctly'].cumsum()
user_df['user_id_roll_count'] = grp['answered_correctly'].cumcount() +1
user_df = user_df.groupby(['user_id',] ).tail(1)
 user_df.set_index('user_id',inplace=True)
 del user_df['answered_correctly']
win_cols = []
for win in [50,10]:
sum_col = f'user_id_win{win}_sum'
count_col = f'user_id_win{win}_count'
user_df[sum_col] = user_df.user_id_roll_sum
user_df[count_col] = user_df.user_id_roll_count
win_cols.append(sum_col)
 win_cols.append(count_col)
 win_current_state = user_df[win_cols]
user_df = add_previous_state(user_df,('user_id',),prev_cache)
 for win in [50,10]:
sum_col = f'user_id_win{win}_sum'
count_col = f'user_id_win{win}_count'
selector =(user_df[count_col]>win ).values
user_df.loc[selector,[sum_col,count_col]] = win_current_state.loc[selector,[sum_col,count_col]]
tmp_df = prev_test_df[['user_answer','answered_correctly','user_id','timestamp']].groupby('user_id' ).tail(1)
 tmp_df.set_index('user_id',inplace=True)
 tmp_df.columns = [f'prev_{col}' for col in tmp_df.columns]
user_df = user_df.merge(tmp_df,how='left',on=['user_id',])
 worker = Worker(update_metadata,(user_df,['user_id'],))
worker.start()
workers.append(worker)
 cols = ['content_id','bundle_id',]
metadata = user_roll_features(prev_test_df,cols)
 for key in metadata:
col = key[-1]
tmp_df = metadata[key]
win_current_state = tmp_df
tmp_df = add_previous_state(tmp_df,key,prev_cache)
 for win in [50,10]:
sum_col = f'{col}_win{win}_sum'
count_col = f'{col}_win{win}_count'
selector =(tmp_df[count_col]>win ).values
tmp_df.loc[selector,[sum_col,count_col]] = win_current_state.loc[selector,[sum_col,count_col]]
if key ==('user_id','content_id'):
tmp = prev_test_df[['content_id','user_id','answered_correctly',]]
tmp.rename(columns={'answered_correctly':'prev_content_answered_correctly'},inplace=True)
 tmp = tmp.groupby(['content_id','user_id',] ).tail(1)
 tmp.set_index(['user_id','content_id'],inplace=True)
 tmp_df = tmp_df.merge(tmp,how='left',on=['user_id','content_id'])
 worker = Worker(update_metadata,(tmp_df,key))
workers.append(worker)
 worker.start()
for worker in workers:
worker.join()
prev_test_df = test_df
prev_cache = cache
full_df.to_pickle("full_df")
 df_train_final.to_pickle("df_train_final")
 df_test_final.to_pickle("df_test_final" )<normalization>
%%skip not SUBMIT
env = riiideducation.make_env()
iter_test = env.iter_test()
prev_test_df = None
prev_cache = None
for(test_df, sample_prediction_df)in iter_test:
test_df = test_df.merge(content_df,how='left',on=['content_id',])
 test_df.prior_question_had_explanation = test_df.prior_question_had_explanation.fillna(False ).astype(np.int8)
 test_state = test_df.loc[test_df['content_type_id'] == 0]
test_state.replace([np.inf, -np.inf], 999999999,inplace=True)
 test_state,cache = merge_metadata(test_state)
 test_state['prior_duration'] = test_state.timestamp-test_state.prev_timestamp
test_state = add_mean(test_state)
 test_state['answered_correctly'] = model.predict(test_state[features])
 if prev_test_df is not None:
workers = []
prior_group_answers_correct	= json.loads(test_df.iloc[0].prior_group_answers_correct)
 prior_group_responses = json.loads(test_df.iloc[0].prior_group_responses)
 prior_group_responses = np.array(prior_group_responses)
 prior_group_answers_correct = np.array(prior_group_answers_correct)
 prev_test_df['user_answer'] = prior_group_responses
prev_test_df['answered_correctly'] = prior_group_answers_correct
prev_test_df = prev_test_df[prev_test_df.content_type_id == 0]
user_df = prev_test_df[['user_id','answered_correctly']]
grp = user_df.groupby(['user_id'])
 user_df['user_id_roll_sum'] = grp['answered_correctly'].cumsum()
user_df['user_id_roll_count'] = grp['answered_correctly'].cumcount() +1
user_df = user_df.groupby(['user_id',] ).tail(1)
 user_df.set_index('user_id',inplace=True)
 del user_df['answered_correctly']
win_cols = []
for win in [50,10]:
sum_col = f'user_id_win{win}_sum'
count_col = f'user_id_win{win}_count'
user_df[sum_col] = user_df.user_id_roll_sum
user_df[count_col] = user_df.user_id_roll_count
win_cols.append(sum_col)
 win_cols.append(count_col)
 win_current_state = user_df[win_cols]
user_df = add_previous_state(user_df,('user_id',),prev_cache)
 for win in [50,10]:
sum_col = f'user_id_win{win}_sum'
count_col = f'user_id_win{win}_count'
selector =(user_df[count_col]>win ).values
user_df.loc[selector,[sum_col,count_col]] = win_current_state.loc[selector,[sum_col,count_col]]
tmp_df = prev_test_df[['user_answer','answered_correctly','user_id','timestamp']].groupby('user_id' ).tail(1)
 tmp_df.set_index('user_id',inplace=True)
 tmp_df.columns = [f'prev_{col}' for col in tmp_df.columns]
user_df = user_df.merge(tmp_df,how='left',on=['user_id',])
 worker = Worker(update_metadata,(user_df,['user_id'],))
worker.start()
workers.append(worker)
 cols = ['content_id','bundle_id',]
metadata = user_roll_features(prev_test_df,cols)
 for key in metadata:
col = key[-1]
tmp_df = metadata[key]
win_current_state = tmp_df
tmp_df = add_previous_state(tmp_df,key,prev_cache)
 for win in [50,10]:
sum_col = f'{col}_win{win}_sum'
count_col = f'{col}_win{win}_count'
selector =(tmp_df[count_col]>win ).values
tmp_df.loc[selector,[sum_col,count_col]] = win_current_state.loc[selector,[sum_col,count_col]]
if key ==('user_id','content_id'):
tmp = prev_test_df[['content_id','user_id','answered_correctly',]]
tmp.rename(columns={'answered_correctly':'prev_content_answered_correctly'},inplace=True)
 tmp = tmp.groupby(['content_id','user_id',] ).tail(1)
 tmp.set_index(['user_id','content_id'],inplace=True)
 tmp_df = tmp_df.merge(tmp,how='left',on=['user_id','content_id'])
 worker = Worker(update_metadata,(tmp_df,key))
workers.append(worker)
 worker.start()
for worker in workers:
worker.join()
prev_test_df = test_df
prev_cache = cache
env.predict(test_state[['row_id', 'answered_correctly']])