Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<load_from_csv>
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 train_data.head()<load_from_csv>
import pandas as pd
import numpy as np
import datatable as dt
from sklearn.metrics import roc_auc_score
from matplotlib import pyplot as plt
import riiideducation
from collections import defaultdict
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import StandardScaler
import tensorflow as tf
from tensorflow.keras import Sequential
from tensorflow.keras.layers import BatchNormalization,Dropout,Dense,Flatten,Conv1D
from tensorflow.keras.optimizers import Adam
from keras.metrics import BinaryAccuracy
from keras import backend as K
test_data = pd.read_csv("/kaggle/input/titanic/test.csv")
 test_data.head()<define_variables>
data_types_dict = {
'user_id': 'int32',
'content_id': 'int16',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'bool'
}
target = 'answered_correctly'
women = train_data.loc[train_data.Sex == 'female']["Survived"]
rate_women = sum(women)/len(women)
 print("% of women who survived:", rate_women )<define_variables>
train_df = dt.fread('.. /input/riiid-test-answer-prediction/train.csv', columns=set(data_types_dict.keys())).to_pandas()
train_df = train_df[train_df[target] != -1].reset_index(drop=True)
 train_df['prior_question_had_explanation'].fillna(False, inplace=True)
 train_df = train_df.astype(data_types_dict )
men = train_data.loc[train_data.Sex == 'male']["Survived"]
rate_men = sum(men)/len(men)
 print("% of men who survived:", rate_men )<count_missing_values>
train_df['lag'] = train_df.groupby('user_id')[target].shift()
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 train_df['user_correctness'] = cum['cumsum'] / cum['cumcount']
train_df.drop(columns=['lag'], inplace=True )
print(train_data.isna().sum())
 print(test_data.isna().sum() )<import_modules>
user_agg = train_df.groupby('user_id')[target].agg(['sum', 'count'])
 content_agg = train_df.groupby('content_id')[target].agg(['sum', 'count'] )
import seaborn as sns
import matplotlib.pyplot as plt<count_values>
questions_df = pd.read_csv(
 '.. /input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 3],
dtype={'question_id': 'int16', 'part': 'int8'} 
)
 train_df = pd.merge(train_df, questions_df, left_on='content_id', right_on='question_id', how='left')
 train_df.drop(columns=['question_id'], inplace=True )
train_data['Survived'].value_counts()<count_values>
train_df['content_count'] = train_df['content_id'].map(content_agg['count'] ).astype('int32')
 train_df['content_avg'] = train_df['content_id'].map(content_agg['sum'] / content_agg['count'] )
train_data['SibSp'].value_counts()<count_values>
valid_df = train_df.groupby('user_id' ).tail(6)
 last_df = train_df.groupby('user_id' ).tail(1)
 train_df.drop(valid_df.index, inplace=True )
train_data['Parch'].value_counts()<count_values>
lencoder = LabelEncoder()
result_time_mean = train_df.prior_question_elapsed_time.mean()
train_df['prior_question_had_explanation'].fillna(False, inplace = True)
 train_df['prior_question_had_explanation_enc'] = lencoder.fit_transform(train_df['prior_question_had_explanation'])
 train_df['prior_question_elapsed_time'].fillna(result_time_mean, inplace = True)
 train_df['user_correctness'].fillna(0.7, inplace=True)
 train_df['part'].fillna(4, inplace=True)
 valid_df['prior_question_had_explanation'].fillna(False, inplace = True)
 valid_df['prior_question_had_explanation_enc'] = lencoder.fit_transform(valid_df['prior_question_had_explanation'])
 valid_df['prior_question_elapsed_time'].fillna(result_time_mean, inplace = True)
 valid_df['user_correctness'].fillna(0.7, inplace=True)
 valid_df['part'].fillna(4, inplace=True)
 last_df['prior_question_had_explanation'].fillna(False, inplace = True)
 last_df['prior_question_had_explanation_enc'] = lencoder.fit_transform(last_df['prior_question_had_explanation'])
 last_df['prior_question_elapsed_time'].fillna(result_time_mean, inplace = True)
 last_df['user_correctness'].fillna(0.7, inplace=True)
 last_df['part'].fillna(4, inplace=True)
print(train_data['Name'].value_counts())
 print(train_data['Ticket'].value_counts())
 print(train_data['Fare'].value_counts() )<count_missing_values>
last_df.to_csv('cnn_last_data.csv',index=True )
print(train_data['Age'].isna().sum())
 print(test_data['Age'].isna().sum() )<data_type_conversions>
X = train_df[['prior_question_elapsed_time','prior_question_had_explanation_enc','part','user_correctness','content_count','content_avg']]
X_val = valid_df[['prior_question_elapsed_time','prior_question_had_explanation_enc','part','user_correctness','content_count','content_avg']]
y = train_df[target]
y_val = valid_df[target]
train_data['Age'].fillna(value=train_data['Age'].median() , inplace=True)
 print(train_data['Age'].isna().sum())
 test_data['Age'].fillna(value=test_data['Age'].median() , inplace=True)
 print(test_data['Age'].isna().sum() )<filter>
X.isnull().any()
train_data = train_data[train_data['Fare'] < 100]<count_missing_values>
scaler = StandardScaler()
X = scaler.fit_transform(X)
 X_val = scaler.transform(X_val )
test_data['Fare'].isna().sum()<count_missing_values>
K.clear_session()
X_train = X.reshape(X.shape[0], X.shape[1], 1)
 X_test = X_val.reshape(X_val.shape[0], X_val.shape[1], 1)
 model = Sequential()
model.add(Conv1D(64, 2, activation='relu', input_shape=X_train[0].shape))
model.add(Conv1D(64, 2, activation='relu', padding='causal'))
model.add(Dropout(0.1))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(Dense(64, activation='relu'))
model.add(Dense(64, activation='relu'))
model.add(Dense(64, activation='relu'))
model.add(Dense(32, activation='relu'))
model.add(Dropout(0.1))
model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer=Adam(learning_rate=0.01), loss='binary_crossentropy', metrics=[tf.keras.metrics.BinaryAccuracy() ] )
train_data['Embarked'].isna().sum()<data_type_conversions>
history = model.fit(X_train, y , epochs=30, verbose=2, batch_size=50000 )
train_data['Embarked'].fillna(method = 'ffill' , inplace = True)
<prepare_x_and_y>
model.save('CNN.model' )
y = train_data['Survived']
features = ['Pclass', 'Sex', 'SibSp', 'Parch']
X = pd.get_dummies(train_data[features])
 X_test = pd.get_dummies(test_data[features])
 X = pd.concat([train_data[['Age', 'Fare']], X], axis=1)
 X_test = pd.concat([test_data[['Age','Fare']], X_test], axis=1)
 print(X)
 print(X_test)
<import_modules>
user_sum_dict = user_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_count_dict = user_agg['count'].astype('int16' ).to_dict(defaultdict(int))
content_sum_dict = content_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
content_count_dict = content_agg['count'].astype('int32' ).to_dict(defaultdict(int))
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score<split>
joblib.dump(user_sum_dict, 'user_sum_dict.pkl')
 joblib.dump(user_count_dict, 'user_count_dict.pkl')
 joblib.dump(content_sum_dict, 'content_sum_dict.pkl')
 joblib.dump(content_count_dict, 'content_count_dict.pkl')
 joblib.dump(lencoder,'exp_encoder.pkl')
 joblib.dump(scaler,'cnn_scaler.pkl')
<save_to_csv><EOS>