Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<create_dataframe>
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1
pd.concat(( pd.DataFrame(train.iloc[:, 1:].columns, columns = ['variable']),
pd.DataFrame(rf.feature_importances_, columns = ['importance'])) ,
axis = 1 ).sort_values(by='importance', ascending = False)[:20]<save_to_csv>
np.seterr(divide = 'ignore', invalid = 'ignore' )
predictions = rf.predict(test)
 predictions = pd.DataFrame(predictions, columns=['Survived'])
 test = pd.read_csv(".. /input/titanic/test.csv")
 predictions = pd.concat(( test.iloc[:, 0], predictions), axis = 1)
 predictions.to_csv('submission.csv', sep=",", index = False )<init_hyperparams>
data_types_dict = {
'user_id': 'int32',
'content_id': 'int16',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'bool'
}
target = 'answered_correctly'
DT=1
optimize_model=False
Make_submission=True
n_estimators=200
max_depth=2<load_from_csv>
train_df = dt.fread('.. /input/riiid-test-answer-prediction/train.csv', columns = set(data_types_dict.keys())).to_pandas()
train=pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/train.csv")
 train.head()<count_unique_values>
train_df = train_df[train_df[target] != -1].reset_index(drop = True, inplace = False)
 train_df['prior_question_had_explanation'].fillna(False, inplace = True)
 train_df = train_df.astype(data_types_dict )
train[train['Province_State'].notna() ].groupby(['Country_Region'], sort=False)['Province_State'].nunique()<data_type_conversions>
train_df['lag'] = train_df.groupby('user_id')[target].shift()
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 train_df['user_correctness'] = cum['cumsum'] / cum['cumcount']
train_df.drop(columns = ['lag'], inplace = True )
def add_location(df_old):
df=df_old.copy()
df['Date']=pd.to_datetime(df['Date'])
 df['Country_Region']=df['Country_Region'].fillna('')
 df['Province_State']=df['Province_State'].fillna('')
 df['location']=df['Province_State'].astype('str')+" "+df['Country_Region'].astype('str')
 return df<feature_engineering>
user_agg = train_df.groupby('user_id')[target].agg(['sum', 'count'])
 content_agg = train_df.groupby('content_id')[target].agg(['sum', 'count'] )
train=add_location(train )<groupby>
train_df = train_df.groupby('user_id' ).tail(24 ).reset_index(drop = True )
max_cases_old=train[train['Date']<'2020-03-22'].groupby(['location'], sort=False)['ConfirmedCases'].max()
max_cases=train.groupby(['location'], sort=False)['ConfirmedCases'].max()
print("Now: {}\r
Seven Days ago: {}".format(len(max_cases[max_cases<5]),len(max_cases_old[max_cases_old<5])) )<feature_engineering>
questions_df = pd.read_csv(
 '.. /input/riiid-test-answer-prediction/questions.csv',
usecols = [0, 3],
dtype = {'question_id': 'int16', 'part': 'int8'} 
)
 train_df = pd.merge(train_df, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left')
 train_df.drop(columns = ['question_id'], inplace = True )
train.set_index('location',inplace=True)
 train['day_of_year']=train['Date'].dt.dayofyear
train['day_of_week']=train['Date'].dt.dayofweek
first_day=train[(train['ConfirmedCases']>0)].groupby(['location'], sort=False)['day_of_year'].min()
first_day.rename('first_day',inplace=True )<feature_engineering>
train_df['content_count'] = train_df['content_id'].map(content_agg['count'] ).astype('int32')
 train_df['content_id'] = train_df['content_id'].map(content_agg['sum'] / content_agg['count'] )
def add_days_passed(df_old,first_day):
df=df_old.copy()
df=pd.concat([df,first_day],axis=1,join='inner')
 df['days_passed']=df['day_of_year']-df['first_day']
df.drop(columns=['first_day'],inplace=True)
 df['location']=df.index
df.set_index('Id',inplace=True)
 df['Id']=df.index
return df 
<feature_engineering>
valid_df = train_df.groupby('user_id' ).tail(6)
 train_df.drop(valid_df.index, inplace = True )
train=add_days_passed(train,first_day)
 train.head()<feature_engineering>
features = ['content_id', 'prior_question_elapsed_time',
'prior_question_had_explanation', 'user_correctness',
'part', 'content_count']
params = {
'loss_function': 'Logloss',
'eval_metric': 'AUC',
'task_type': 'GPU' if torch.cuda.is_available() else 'CPU',
'grow_policy': 'Lossguide',
'iterations': 1000,
'learning_rate': 4e-3,
'random_seed': 0,
'l2_leaf_reg': 1e-1,
'depth': 15,
'max_leaves': 13,
'border_count': 128,
'verbose': 50,
}
params1 = {
'objective': 'binary',
'seed': 42,
'metric': 'auc',
'learning_rate': 0.05,
'max_bin': 800,
'num_leaves': 85
}
country_stat=pd.read_csv('.. /input/countryinfo/covid19countryinfo.csv')
 country_stat = country_stat[country_stat['region'].isnull() ]
def add_country_stat(old_df,country_stat):
df=old_df.copy()
df=df.merge(country_stat[['country','pop','medianage','sex65plus','lung','smokers','density']],left_on=['Country_Region'],right_on=['country'],how='left')
 df.drop(columns=['country'],inplace=True)
 df['pop']=df['pop'].fillna(1000)
 df['pop']=df['pop'].apply(lambda x: int(str(x ).replace(',', '')))
 df['density']=df['density'].fillna(0)
 df['medianage']=df['medianage'].fillna(0)
 df['sex65plus']=df['sex65plus'].fillna(1)
 df['lung']=df['lung'].fillna(24)
 df['smokers']=df['smokers'].fillna(24)
 return df
train=add_country_stat(train,country_stat )<load_from_csv>
tr_data = lgb.Dataset(train_df[features], label=train_df[target])
 va_data = lgb.Dataset(valid_df[features], label=valid_df[target])
 model1 = lgb.train(
 params1,
tr_data,
num_boost_round=10000,
valid_sets=[tr_data, va_data],
early_stopping_rounds=50,
verbose_eval=50 
)
 lgb.plot_importance(model1, importance_type='gain')
 plt.show()
border_info=pd.read_csv("https://raw.githubusercontent.com/geodatasource/country-borders/master/GEODATASOURCE-COUNTRY-BORDERS.CSV")
 border_info.drop(columns=["country_code","country_border_code"],inplace=True)
 border_info.replace({'United States of America':'US',
'United Kingdom of Great Britain and Northern Ireland':'United Kingdom',
'Bolivia(Plurinational State Of)':'Bolivia',
'Brunei Darussalam':'Brunei',
'Gambia(the)':'Gambia',
'Congo':'Congo(Kinshasa)',
'Cote d’Ivoire':"Cote d'Ivoire",
"Iran(Islamic Republic of)":'Iran',
"Korea(the Republic of)":'Korea, South',
"Lao People's Democratic Republic":'Laos',
"Moldova(the Republic of)":'Moldova',
"Russian Federation":'Russia',
"Syrian Arab Republic":'Syria',
"Taiwan(Province of China)":'Taiwan*',
"Tanzania(the United Republic of)":'Tanzania',
"Venezuela(Bolivarian Republic of)":'Venezuela',
"Viet Nam":'Vietnam'},inplace=True)
 border_info=border_info.fillna("")
 border_info.to_csv("border_info.csv")
<create_dataframe>
train_set = Pool(train_df[features], label = train_df[target])
 val_set = Pool(valid_df[features], label = valid_df[target] )
def expand_grid(data_dict):
rows = it_product(*data_dict.values())
 return pd.DataFrame.from_records(rows, columns=data_dict.keys() )<groupby>
model = CatBoostClassifier(**params)
 model.fit(train_set, eval_set = val_set, use_best_model = True )
country_info=train.groupby(['Date','Country_Region'])['ConfirmedCases'].sum()<merge>
user_sum_dict = user_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_count_dict = user_agg['count'].astype('int16' ).to_dict(defaultdict(int))
content_sum_dict = content_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
content_count_dict = content_agg['count'].astype('int32' ).to_dict(defaultdict(int))
skel=expand_grid({'Index':border_info.index,'Date':train['Date'].unique() })
 skel=skel.merge(border_info, how='inner', left_on=['Index'],right_index=True)
 skel=skel.merge(country_info, how='inner',
left_on=['Date','country_border_name'],right_on=['Date','Country_Region'] )<feature_engineering>
try:
env = riiideducation.make_env()
except:
pass
iter_test = env.iter_test()
prior_test_df = None
<merge><EOS> 
%%time
for(test_df, sample_prediction_df)in iter_test:
if prior_test_df is not None:
prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop = True)
 user_ids = prior_test_df['user_id'].values
content_ids = prior_test_df['content_id'].values
targets = prior_test_df[target].values
for user_id, content_id, answered_correctly in zip(user_ids, content_ids, targets):
user_sum_dict[user_id] += answered_correctly
user_count_dict[user_id] += 1
content_sum_dict[content_id] += answered_correctly
content_count_dict[content_id] += 1
prior_test_df = test_df.copy()
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop = True)
 test_df = pd.merge(test_df, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left')
 test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].fillna(False ).astype('bool')
 user_sum = np.zeros(len(test_df), dtype = np.int16)
 user_count = np.zeros(len(test_df), dtype = np.int16)
 content_sum = np.zeros(len(test_df), dtype = np.int32)
 content_count = np.zeros(len(test_df), dtype = np.int32)
 for i,(user_id, content_id)in enumerate(zip(test_df['user_id'].values, test_df['content_id'].values)) :
user_sum[i] = user_sum_dict[user_id]
user_count[i] = user_count_dict[user_id]
content_sum[i] = content_sum_dict[content_id]
content_count[i] = content_count_dict[content_id]
test_df['user_correctness'] = user_sum / user_count
test_df['content_count'] = content_count
test_df['content_id'] = content_sum / content_count
test_df[target] = np.average([
model.predict_proba(test_df[features].values)[:,1],
model1.predict(test_df[features])
 ], weights=[0.49934,0.50066], axis=0)
 env.predict(test_df[['row_id', target]] )