Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<feature_engineering>
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1
all_df['Relatives'] = all_df['SibSp'] + all_df['Parch']<filter>
_ = np.seterr(divide='ignore', invalid='ignore' )
all_df[all_df['Embarked'].isnull() ]<sort_values>
data_types_dict = {
'user_id': 'int32',
'content_id': 'int16',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'bool'
}
target = 'answered_correctly'
sort_embarked_features = ['Cabin', 'Sex', 'Pclass', 'Alone', 'Fare']
sorted_df = all_df.sort_values(by=sort_embarked_features)
 sorted_df[(sorted_df['Survived'] == 1)& 
(sorted_df['Sex'] == 'female')& 
(sorted_df['Pclass'] == 1)& 
(sorted_df['Alone'] == 1)].sort_values(by=sort_embarked_features)[sort_embarked_features+['Embarked']].head()<data_type_conversions>
train_df = dt.fread('.. /input/riiid-test-answer-prediction/train.csv', columns=set(data_types_dict.keys())).to_pandas()
train_df = train_df[train_df[target] != -1].reset_index(drop=True)
 train_df['prior_question_had_explanation'].fillna(False, inplace=True)
 train_df = train_df.astype(data_types_dict )
all_df['Embarked'].fillna(value='C', inplace=True )<filter>
train_df['lag'] = train_df.groupby('user_id')[target].shift()
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 train_df['user_correctness'] = cum['cumsum'] / cum['cumcount']
train_df.drop(columns=['lag'], inplace=True )
all_df[all_df['Fare'].isnull() ]<sort_values>
user_agg = train_df.groupby('user_id')[target].agg(['sum', 'count'])
 content_agg = train_df.groupby('content_id')[target].agg(['sum', 'count'] )
sort_fare_features = ['Sex', 'Pclass', 'Title', 'Deck', 'Embarked', 'Alone', 'Age']
sorted_df = all_df.sort_values(by=sort_embarked_features)
 aux = sorted_df[(sorted_df['Alone'] == 1)& 
(sorted_df['Sex'] == 'male')& 
(sorted_df['Pclass'] == 1)& 
(sorted_df['Embarked'] == 'S')& 
(sorted_df['Deck'] == 'N')& 
(sorted_df['Title'] == 'Mr')& 
(( sorted_df['Age'] >= 55)&(sorted_df['Age'] <= 65)) ]
aux<data_type_conversions>
questions_df = pd.read_csv(
 '.. /input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 3],
dtype={'question_id': 'int16', 'part': 'int8'} 
)
 train_df = pd.merge(train_df, questions_df, left_on='content_id', right_on='question_id', how='left')
 train_df.drop(columns=['question_id'], inplace=True )
all_df['Fare'].fillna(value=aux['Fare'].mean() , inplace=True )<statistical_test>
train_df['content_count'] = train_df['content_id'].map(content_agg['count'] ).astype('int32')
 train_df['content_id'] = train_df['content_id'].map(content_agg['sum'] / content_agg['count'] )
def impute_num(cols, avg, std):
try:
avg_value = avg.loc[tuple(cols)][0]
except Exception as e:
print(f'It is not possible to find an average value for this combination of features values:
{cols}')
 return np.nan
try:
std_value = std.loc[tuple(cols)][0]
except Exception as e:
std_value = 0
finally:
if pd.isnull(std_value):
std_value = 0
while True:
value = np.random.randint(avg_value-std_value, avg_value+std_value+1)
 if value >= 0:
break
return round(value, 0 )<define_variables>
valid_df = train_df.groupby('user_id' ).tail(6)
 train_df.drop(valid_df.index, inplace=True )
group_age_features = ['Title','Relatives','Parch','SibSp','Deck','Pclass','Embarked','Sex','Alone']<create_dataframe>
features = [
'content_id',
'prior_question_elapsed_time',
'prior_question_had_explanation',
'user_correctness',
'part',
'content_count'
]
params = {
'objective': 'binary',
'seed': 42,
'metric': 'auc',
'learning_rate': 0.05,
'max_bin': 800,
'num_leaves': 80
}
stat_age = all_df.pivot_table(values='Age', index=group_age_features, aggfunc=['mean','std'] ).round(2 )<prepare_output>
tr_data = lgb.Dataset(train_df[features], label=train_df[target])
 va_data = lgb.Dataset(valid_df[features], label=valid_df[target])
 model = lgb.train(
 params,
tr_data,
num_boost_round=10000,
valid_sets=[tr_data, va_data],
early_stopping_rounds=50,
verbose_eval=50 
)
 lgb.plot_importance(model, importance_type='gain')
 plt.show()
comp = pd.DataFrame([all_df[~all_df['Age'].isnull() ]['Age'], ages2, ages1] ).T
comp.columns = ['Real Age', 'Predicted Age $(\mu=\mu_{Age}$, $\sigma=\sigma_{Age})$', 'Predicted Age $(\mu=\mu_{Age}$, $\sigma=0)$']
comp.head(10 )<compute_test_metric>
user_sum_dict = user_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_count_dict = user_agg['count'].astype('int16' ).to_dict(defaultdict(int))
content_sum_dict = content_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
content_count_dict = content_agg['count'].astype('int32' ).to_dict(defaultdict(int))
<define_variables><EOS> 
%%time
for(test_df, sample_prediction_df)in iter_test:
if prior_test_df is not None:
prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)
 user_ids = prior_test_df['user_id'].values
content_ids = prior_test_df['content_id'].values
targets = prior_test_df[target].values
for user_id, content_id, answered_correctly in zip(user_ids, content_ids, targets):
user_sum_dict[user_id] += answered_correctly
user_count_dict[user_id] += 1
content_sum_dict[content_id] += answered_correctly
content_count_dict[content_id] += 1
prior_test_df = test_df.copy()
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df = pd.merge(test_df, questions_df, left_on='content_id', right_on='question_id', how='left')
 test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].fillna(False ).astype('bool')
 user_sum = np.zeros(len(test_df), dtype=np.int16)
 user_count = np.zeros(len(test_df), dtype=np.int16)
 content_sum = np.zeros(len(test_df), dtype=np.int32)
 content_count = np.zeros(len(test_df), dtype=np.int32)
 for i,(user_id, content_id)in enumerate(zip(test_df['user_id'].values, test_df['content_id'].values)) :
user_sum[i] = user_sum_dict[user_id]
user_count[i] = user_count_dict[user_id]
content_sum[i] = content_sum_dict[content_id]
content_count[i] = content_count_dict[content_id]
test_df['user_correctness'] = user_sum / user_count
test_df['content_count'] = content_count
test_df['content_id'] = content_sum / content_count
test_df[target] = model.predict(test_df[features])
 env.predict(test_df[['row_id', target]] )