Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<choose_model_class>
env = riiideducation.make_env()
class LeNet:
	@staticmethod
	def build(input_shape, classes):
		model = Sequential()
		
		model.add(Conv2D(20, kernel_size=5, padding="same",
			input_shape=input_shape))
		model.add(Activation("relu"))
		model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
 		
		model.add(Conv2D(50, kernel_size=5, padding="same"))
		model.add(Activation("relu"))
		model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
 		
		model.add(Flatten())
 		model.add(Dense(500))
		model.add(Dropout(0.3))
		model.add(Activation("relu"))
		
		model.add(Dense(classes))
		model.add(Activation("softmax"))
		return model 
<define_search_space>
train = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',
usecols=[1, 2, 3, 4, 7, 8, 9],
dtype={'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id': 'int8',
'answered_correctly':'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'boolean'}
)
NB_EPOCH = 20
BATCH_SIZE = 128
VERBOSE = 1
OPTIMIZER = Adam()
VALIDATION_SPLIT=0.2
IMG_ROWS, IMG_COLS = 28, 28
NB_CLASSES = 10
INPUT_SHAPE =(IMG_ROWS, IMG_COLS, 1)
<choose_model_class>
train = train[train.content_type_id == False]
train = train.sort_values(['timestamp'], ascending=True ).reset_index(drop = True)
 train.head(10 )
model = LeNet.build(input_shape=INPUT_SHAPE, classes=NB_CLASSES)
 model.compile(loss="categorical_crossentropy", optimizer=OPTIMIZER,
	metrics=["accuracy"])
<train_model>
results_c_final = train[['content_id','answered_correctly']].groupby(['content_id'] ).agg(['mean'])
 results_c_final.columns = ["answered_correctly_content"]
results_u_final = train[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean', 'sum', 'count'])
 results_u_final.columns = ['answered_correctly_user', 'sum', 'count']
history = model.fit(X_train, Y_train,
		batch_size=BATCH_SIZE, epochs=NB_EPOCH,
		verbose=VERBOSE, validation_split=VALIDATION_SPLIT)
<split>
train.loc[(train.timestamp < 1000000)&(train.timestamp > 0)].answered_correctly.mean()
lr_reduction = ReduceLROnPlateau(monitor='val_accuracy', patience=3, verbose=1, factor=0.5, min_lr=0.00001)
 X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.1, random_state=2)
 datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )<train_model>
train.prior_question_had_explanation.value_counts()
NB_EPOCH = 30
BATCH_SIZE = 128
VERBOSE = 1
model.compile(loss="categorical_crossentropy", optimizer=OPTIMIZER,
	metrics=["accuracy"])
 history = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=BATCH_SIZE),
epochs = NB_EPOCH, validation_data =(X_val,Y_val),
verbose = VERBOSE, steps_per_epoch=X_train.shape[0] // BATCH_SIZE
, callbacks=[lr_reduction] )<predict_on_test>
train.drop(['timestamp', 'content_type_id'], axis=1, inplace=True )
results = model.predict(test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label")
<save_to_csv>
validation = pd.DataFrame()
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("LeNet_mnist_datagen.csv",index=False)
<set_options>
for i in range(4):
last_records = train.drop_duplicates('user_id', keep = 'last')
 train = train[~train.index.isin(last_records.index)]
validation = validation.append(last_records )
%matplotlib inline<compute_test_metric>
X = pd.DataFrame()
def dS_dt(S, I, R_t, T_inf):
return -(R_t / T_inf)* I * S
def dE_dt(S, E, I, R_t, T_inf, T_inc):
return(R_t / T_inf)* I * S -(T_inc**-1)* E
def dI_dt(I, E, T_inc, T_inf):
return(T_inc**-1)* E -(T_inf**-1)* I
def dR_dt(I, T_inf):
return(T_inf**-1)* I
def SEIR_model(t, y, R_t, T_inf, T_inc):
if callable(R_t):
reproduction = R_t(t)
 else:
reproduction = R_t
S, E, I, R = y
S_out = dS_dt(S, I, reproduction, T_inf)
 E_out = dE_dt(S, E, I, reproduction, T_inf, T_inc)
 I_out = dI_dt(I, E, T_inc, T_inf)
 R_out = dR_dt(I, T_inf)
 return [S_out, E_out, I_out, R_out]<load_from_csv>
for i in range(15):
last_records = train.drop_duplicates('user_id', keep = 'last')
 train = train[~train.index.isin(last_records.index)]
X = X.append(last_records )
train = pd.read_csv('.. /input/covid19-global-forecasting-week-3/train.csv')
 test = pd.read_csv('.. /input/covid19-global-forecasting-week-3/test.csv')
 train['Date_datetime'] = train['Date'].apply(lambda x:(datetime.datetime.strptime(x, '%Y-%m-%d')) )<load_from_csv>
results_c = train[['content_id','answered_correctly']].groupby(['content_id'] ).agg(['mean'])
 results_c.columns = ["answered_correctly_content"]
results_u = train[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean', 'sum', 'count'])
 results_u.columns = ["answered_correctly_user", 'sum', 'count']
pop_info = pd.read_csv('/kaggle/input/covid19-population-data/population_data.csv')
 country_pop = pop_info.query('Type == "Country/Region"')
 province_pop = pop_info.query('Type == "Province/State"')
 country_lookup = dict(zip(country_pop['Name'], country_pop['Population']))
province_lookup = dict(zip(province_pop['Name'], province_pop['Population']))<import_modules>
del(train)
from scipy.optimize import minimize
from sklearn.metrics import mean_squared_log_error, mean_squared_error<compute_train_metric>
X = pd.merge(X, results_u, on=['user_id'], how="left")
 X = pd.merge(X, results_c_final, on=['content_id'], how="left" )
T_inc = 5.2
T_inf = 2.9
R_0, cfr, k, L=[ 3.95469597 , 0.04593316 , 3., 15.32328881]
def time_varying_reproduction(t):
return R_0 /(1 +(t/L)**k )<compute_train_metric>
validation = pd.merge(validation, results_u, on=['user_id'], how="left")
 validation = pd.merge(validation, results_c_final, on=['content_id'], how="left" )
def eval_model_const(params, data, population, return_solution=False, forecast_days=0):
R_0, cfr = params
N = population
n_infected = data['ConfirmedCases'].iloc[0]
max_days = len(data)+ forecast_days
s, e, i, r =(N - n_infected)/ N, 0, n_infected / N, 0
def time_varying_reproduction(t):
if t > 80:
return R_0 * 0.5
else:
return R_0
sol = solve_ivp(SEIR_model, [0, max_days], [s, e, i, r], args=(time_varying_reproduction, T_inf, T_inc),
t_eval=np.arange(0, max_days))
sus, exp, inf, rec = sol.y
y_pred_cases = np.clip(( inf + rec)* N ,0,np.inf)
 y_true_cases = data['ConfirmedCases'].values
y_pred_fat = np.clip(rec*N* cfr, 0, np.inf)
 y_true_fat = data['Fatalities'].values
optim_days = min(20, len(data))
weights = 1 / np.arange(1, optim_days+1)[::-1]
msle_cases = mean_squared_log_error(y_true_cases[-optim_days:], y_pred_cases[-optim_days:], weights)
 msle_fat = mean_squared_log_error(y_true_fat[-optim_days:], y_pred_fat[-optim_days:], weights)
 msle_final = np.mean([msle_cases, msle_fat])
 if return_solution:
return msle_final, sol
else:
return msle_final<compute_train_metric>
lb_make = LabelEncoder()
X.prior_question_had_explanation.fillna(False, inplace = True)
 validation.prior_question_had_explanation.fillna(False, inplace = True)
 validation["prior_question_had_explanation_enc"] = lb_make.fit_transform(validation["prior_question_had_explanation"])
 X["prior_question_had_explanation_enc"] = lb_make.fit_transform(X["prior_question_had_explanation"])
 X.head()
def eval_model_decay(params, data, population, return_solution=False, forecast_days=0):
R_0, cfr, k, L = params
N = population
n_infected = data['ConfirmedCases'].iloc[0]
max_days = len(data)+ forecast_days
s, e, i, r =(N - n_infected)/ N, 0, n_infected / N, 0
def time_varying_reproduction(t):
return R_0 /(1 +(t/L)**k)
 sol = solve_ivp(SEIR_model, [0, max_days], [s, e, i, r], args=(time_varying_reproduction, T_inf, T_inc),
t_eval=np.arange(0, max_days))
sus, exp, inf, rec = sol.y
y_pred_cases = np.clip(( inf + rec)* N ,0,np.inf)
 y_true_cases = data['ConfirmedCases'].values
y_pred_fat = np.clip(rec*N* cfr, 0, np.inf)
 y_true_fat = data['Fatalities'].values
optim_days = min(20, len(data))
weights = 1 / np.arange(1, optim_days+1)[::-1]
msle_cases = mean_squared_log_error(y_true_cases[-optim_days:], y_pred_cases[-optim_days:], weights)
 msle_fat = mean_squared_log_error(y_true_fat[-optim_days:], y_pred_fat[-optim_days:], weights)
 msle_final = np.mean([msle_cases, msle_fat])
 if return_solution:
return msle_final, sol
else:
return msle_final<train_model>
questions_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 3],
dtype={'question_id': 'int16',
'part': 'int8'}
)
def fit_model_new(data, area_name, initial_guess=[2.2, 0.02, 2, 50],
bounds=(( 1, 20),(0, 0.15),(1, 3),(1, 100)) , make_plot=True, decay_mode = None):
if area_name in ['France']:
train = data.query('ConfirmedCases > 0' ).copy() [:-1]
else:
train = data.query('ConfirmedCases > 0' ).copy()
valid_data = train[-7:].copy()
train_data = train[:-7].copy()
if len(train_data)== 0:
result_zero = np.zeros(( 43))
return pd.DataFrame({'ConfirmedCases':result_zero,'Fatalities':result_zero}), 0
try:
population = pop_info[pop_info['Name']==area_name]['Population'].tolist() [0]
except IndexError:
print('country not in population set, '+str(area_name))
population = 1000000
if area_name == 'US':
population = 327200000
if area_name == 'Global':
population = 7744240900
cases_per_million = train_data['ConfirmedCases'].max() * 10**6 / population
n_infected = train_data['ConfirmedCases'].iloc[0]
if cases_per_million < 1:
population = population/100
res_const = minimize(eval_model_const, [2.2, 0.02], bounds=(( 1, 20),(0, 0.15)) ,
args=(train_data, population, False),
method='L-BFGS-B')
 res_decay = minimize(eval_model_decay, initial_guess, bounds=bounds,
args=(train_data, population, False),
method='L-BFGS-B')
 test_end = datetime.datetime.strptime('2020-05-07','%Y-%m-%d')
 test_start = datetime.datetime.strptime('2020-03-26','%Y-%m-%d')
 test_period =(test_end - test_start ).days
train_max = train_data.Date_datetime.max()
train_all_max = train.Date_datetime.max()
train_min = train_data.Date_datetime.min()
add_date = 0
delta_days =(test_end - train_max ).days
train_add_time=[]
if train_min > test_start:
add_date =(train_min-test_start ).days
last = train_min-timedelta(add_date)
 train_add_time = np.arange(last, train_min, dtype='datetime64[D]' ).tolist()
train_add_time = pd.to_datetime(train_add_time)
 dates_all = train_add_time.append(pd.to_datetime(np.arange(train_min, test_end+timedelta(1), dtype='datetime64[D]')))
 else:
dates_all = pd.to_datetime(np.arange(train_min, test_end+timedelta(1), dtype='datetime64[D]'))
if decay_mode is None:
if res_const.fun < res_decay.fun :
msle, sol = eval_model_const(res_const.x, train_data, population, True, delta_days+add_date)
 res = res_const
else:
msle, sol = eval_model_decay(res_decay.x, train_data, population, True, delta_days+add_date)
 res = res_decay
R_0, cfr, k, L = res.x
else:
if decay_mode =='day_decay':
msle, sol = eval_model_const(res_const.x, train_data, population, True, delta_days+add_date)
 res = res_const
else:
msle, sol = eval_model_decay(res_decay.x, train_data, population, True, delta_days+add_date)
 res = res_decay
R_0, cfr, k, L = res.x
sus, exp, inf, rec = sol.y
y_pred = pd.DataFrame({
'ConfirmedCases': cumsum_signal(np.diff(( inf + rec)* population, prepend=n_infected ).cumsum()),
'Fatalities': cumsum_signal(( np.clip(rec * population * res.x[1], 0, np.inf)).tolist())
 })
 y_pred_valid = y_pred.iloc[len(train_data):len(train_data)+len(valid_data)]
y_pred_test = y_pred.iloc[-(test_period+1):]
y_true_valid = valid_data[['ConfirmedCases', 'Fatalities']]
valid_msle_cases = mean_squared_log_error(y_true_valid['ConfirmedCases'], y_pred_valid['ConfirmedCases'])
 valid_msle_fat = mean_squared_log_error(y_true_valid['Fatalities'], y_pred_valid['Fatalities'])
 valid_msle = np.mean([valid_msle_cases, valid_msle_fat])
 if make_plot:
if len(res.x)<=2:
print(f'Validation MSLE: {valid_msle:0.5f}, using intervention days decay, Reproduction number(R0): {res.x[0]:0.5f}, Fatal rate : {res.x[1]:0.5f}')
 else:
print(f'Validation MSLE: {valid_msle:0.5f}, using Hill decay, Reproduction number(R0): {res.x[0]:0.5f}, Fatal rate : {res.x[1]:0.5f}, K : {res.x[2]:0.5f}, L: {res.x[3]:0.5f}')
 f = plt.figure(figsize=(16,5))
ax = f.add_subplot(1,2,1)
 ax.plot(exp, 'y', label='Exposed');
ax.plot(inf, 'r', label='Infected');
ax.plot(rec, 'c', label='Recovered/deceased');
plt.title('SEIR Model Trends')
 plt.xlabel("Days", fontsize=10);
plt.ylabel("Fraction of population", fontsize=10);
plt.legend(loc='best');
ax2 = f.add_subplot(1,2,2)
 xaxis = train_data['Date_datetime'].tolist()
xaxis = dates.date2num(xaxis)
 hfmt = dates.DateFormatter('%m
%d')
 ax2.xaxis.set_major_formatter(hfmt)
 ax2.plot(np.array(train_data['Date_datetime'], dtype='datetime64[D]'),train_data['ConfirmedCases'],label='Confirmed Cases(train)', c='g')
 ax2.plot(np.array(train_data['Date_datetime'], dtype='datetime64[D]'), y_pred['ConfirmedCases'][:len(train_data)],label='Cumulative modeled infections', c='r')
 ax2.plot(np.array(valid_data['Date_datetime'], dtype='datetime64[D]'), y_true_valid['ConfirmedCases'],label='Confirmed Cases(valid)', c='b')
 ax2.plot(np.array(valid_data['Date_datetime'], dtype='datetime64[D]'),y_pred_valid['ConfirmedCases'],label='Cumulative modeled infections(valid)', c='y')
 plt.title('Real ConfirmedCase and Predict ConfirmedCase')
 plt.legend(loc='best');
plt.show()
if len(res.x)>2:
msle, sol = eval_model_decay(res.x, train_data, population, True, 300)
 else:
msle, sol = eval_model_const(res.x, train_data, population, True, 300)
 sus, exp, inf, rec = sol.y
y_pred = pd.DataFrame({
'ConfirmedCases': cumsum_signal(np.diff(( inf + rec)* population, prepend=n_infected ).cumsum()),
'Fatalities': cumsum_signal(np.clip(rec, 0, np.inf)* population * res.x[1])
 })
 start = train_min
end = start + timedelta(len(y_pred))
time_array = np.arange(start, end, dtype='datetime64[D]')
 max_day = numpy.where(inf == numpy.amax(inf)) [0][0]
where_time = time_array[max_day]
pred_max_day = y_pred['ConfirmedCases'][max_day]
xy_show_max_estimation =(where_time, max_day)
 con = y_pred['ConfirmedCases']
max_day_con = numpy.where(con == numpy.amax(con)) [0][0]
max_con = numpy.amax(con)
 where_time_con = time_array[len(time_array)-50]
xy_show_max_estimation_confirmed =(where_time_con, max_con)
 fig = go.Figure()
fig.add_trace(go.Scatter(x=time_array, y=y_pred['ConfirmedCases'].astype(int),
mode='lines',
line = dict(color='red'),
name='Estimation Confirmed Case Start from '+ str(start.date())+ ' to ' +str(end.date())))
 fig.add_trace(go.Scatter(x=time_array[:len(train)], y=train['ConfirmedCases'],
mode='lines',
name='Confirmed case until '+ str(train_all_max.date()),line = dict(color='green', width=4)))
 fig.add_annotation(
 x=where_time_con,
y=max_con-(max_con/30),
showarrow=False,
text="Estimate Max Case around:" +str(int(max_con)) ,
font=dict(
 color="Blue",
size=15 
))
fig.add_annotation(
 x=time_array[len(train)-1],
y=train['ConfirmedCases'].tolist() [-1],
showarrow=True,
text=f"Real Max ConfirmedCase: " +str(int(train['ConfirmedCases'].tolist() [-1])))
 fig.add_annotation(
 x=where_time,
y=pred_max_day,
text='Infect start decrease from: ' + str(where_time))
fig.update_layout(title='Estimate Confirmed Case ,'+area_name+' Total population ='+ str(int(population)) , legend_orientation="h")
 fig.show()
return y_pred_test, valid_msle<count_unique_values>
X = pd.merge(X, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left')
 validation = pd.merge(validation, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left')
 X.part = X.part - 1
validation.part = validation.part - 1
X.part.value_counts()
validation_scores = []
validation_county = []
validation_country = []
for country in tqdm(train['Country_Region'].unique()):
country_pd_train = train[train['Country_Region']==country]
if len(country_pd_train['Province_State'].unique())<2:
predict_test, score = fit_model_new(country_pd_train,country,make_plot=False)
 if score ==0:
print(f'{country} no case')
 validation_scores.append(score)
 validation_county.append(country)
 validation_country.append(country)
 test.loc[test['Country_Region']==country,'ConfirmedCases'] = predict_test['ConfirmedCases'].tolist()
test.loc[test['Country_Region']==country,'Fatalities'] = predict_test['Fatalities'].tolist()
else:
for state in country_pd_train['Province_State'].unique() :
if state != state:
state_pd = country_pd_train[country_pd_train['Province_State'].isna() ]
predict_test, score = fit_model_new(state_pd,state,make_plot=False)
 if score ==0:
print(f'{country} / {state} no case')
 validation_scores.append(score)
 validation_county.append(state)
 validation_country.append(country)
 test.loc[(test['Country_Region']==country)&(test['Province_State'].isna()),'ConfirmedCases'] = predict_test['ConfirmedCases'].tolist()
test.loc[(test['Country_Region']==country)&(test['Province_State'].isna()),'Fatalities'] = predict_test['Fatalities'].tolist()
else:
state_pd = country_pd_train[country_pd_train['Province_State']==state]
predict_test, score = fit_model_new(state_pd,state,make_plot=False)
 if score ==0:
print(f'{country} / {state} no case')
 validation_scores.append(score)
 validation_county.append(state)
 validation_country.append(country)
 test.loc[(test['Country_Region']==country)&(test['Province_State']==state),'ConfirmedCases'] = predict_test['ConfirmedCases'].tolist()
test.loc[(test['Country_Region']==country)&(test['Province_State']==state),'Fatalities'] = predict_test['Fatalities'].tolist()
print(f'Mean validation score: {np.average(validation_scores):0.5f}' )<prepare_output>
y = X['answered_correctly']
X = X.drop(['answered_correctly'], axis=1)
 X.head()
y_val = validation['answered_correctly']
X_val = validation.drop(['answered_correctly'], axis=1 )
validation_scores = pd.DataFrame({'country/state':validation_country,'country':validation_county,'MSLE':validation_scores})
 validation_scores.sort_values(by=['MSLE'], ascending=False ).head(20 )<filter>
X = X[['answered_correctly_user', 'answered_correctly_content', 'sum', 'count',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part']]
X_val = X_val[['answered_correctly_user', 'answered_correctly_content', 'sum', 'count',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part']]
large_msle = validation_scores[validation_scores['MSLE']>1]<filter>
X_val['answered_correctly_user'].fillna(0.5, inplace=True)
 X_val['answered_correctly_content'].fillna(0.5, inplace=True)
 X_val['part'].fillna(4, inplace = True)
 X_val['count'].fillna(0, inplace = True)
 X_val['sum'].fillna(0, inplace = True)
 X_val['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 X_val['prior_question_had_explanation_enc'].fillna(0, inplace = True )
for country in large_msle['country'].unique() :
if(country!= country)==False:
country_pd_train = train[train['Country_Region']==country]
country_pd_test = test[test['Country_Region']==country]
if len(country_pd_train)==0:
country_pd_train = train[train['Province_State']==country]
country_pd_test = test[test['Province_State']==country]
x = np.array(range(len(country_pd_train)) ).reshape(( -1,1)) [:-7]
valid_x = np.array(range(len(country_pd_train)) ).reshape(( -1,1)) [-7:]
y = country_pd_train['ConfirmedCases'][:-7]
valid_y = country_pd_train['ConfirmedCases'][-7:]
y_fat = country_pd_train['Fatalities'][:-7]
valid_y_fat = country_pd_train['Fatalities'][-7:]
model = Pipeline([('poly', PolynomialFeatures(degree=2)) , 
('linear', LinearRegression(fit_intercept=False)) ])
 model = model.fit(x, y)
 model_fat = Pipeline([('poly', PolynomialFeatures(degree=2)) , 
('linear', LinearRegression(fit_intercept=False)) ])
 model_fat = model_fat.fit(x, y_fat)
 predict_y = model.predict(valid_x)
 predict_yfat = model_fat.predict(valid_x)
 score = mean_squared_log_error(np.clip(valid_y,0,np.inf), np.clip(predict_y,0,np.inf))
score_fat = mean_squared_log_error(np.clip(valid_y_fat,0,np.inf), np.clip(predict_yfat,0,np.inf))
score =(score+score_fat)/2
print(f'{country} {score:0.5f}')
 if score < large_msle[large_msle['country']==country]['MSLE'].tolist() [0]:
validation_scores.loc[validation_scores['country']==country,'MSLE'] = score
predict_x =(np.array(range(len(country_pd_test)))+50 ).reshape(( -1,1))
test.loc[test['Province_State']==country,'ConfirmedCases'] = model.predict(predict_x)
 test.loc[test['Province_State']==country,'Fatalities'] = model_fat.predict(predict_x)
 else:
x = np.array(range(len(country_pd_train)) ).reshape(( -1,1)) [:-7]
valid_x = np.array(range(len(country_pd_train)) ).reshape(( -1,1)) [-7:]
y = country_pd_train['ConfirmedCases'][:-7]
valid_y = country_pd_train['ConfirmedCases'][-7:]
y_fat = country_pd_train['Fatalities'][:-7]
valid_y_fat = country_pd_train['Fatalities'][-7:]
model = Pipeline([('poly', PolynomialFeatures(degree=2)) , 
('linear', LinearRegression(fit_intercept=False)) ])
 model = model.fit(x, y)
 model_fat = Pipeline([('poly', PolynomialFeatures(degree=2)) , 
('linear', LinearRegression(fit_intercept=False)) ])
 model_fat = model_fat.fit(x, y_fat)
 predict_y = model.predict(valid_x)
 predict_yfat = model_fat.predict(valid_x)
 score = mean_squared_log_error(np.clip(valid_y,0,np.inf), np.clip(predict_y,0,np.inf))
score_fat = mean_squared_log_error(np.clip(valid_y_fat,0,np.inf), np.clip(predict_yfat,0,np.inf))
score =(score+score_fat)/2
print(f'{country} {score:0.5f}')
 if score < large_msle[large_msle['country']==country]['MSLE'].tolist() [0]:
validation_scores.loc[validation_scores['country']==country,'MSLE'] = score
predict_x =(np.array(range(len(country_pd_test)))+50 ).reshape(( -1,1))
test.loc[test['Country_Region']==country,'ConfirmedCases'] = model.predict(predict_x)
 test.loc[test['Country_Region']==country,'Fatalities'] = model_fat.predict(predict_x )<compute_test_metric>
params = {
'objective': 'binary',
'max_bin': 700,
'learning_rate': 0.0175,
'num_leaves': 80
}
lgb_train = lgb.Dataset(X, y, categorical_feature = ['part', 'prior_question_had_explanation_enc'])
 lgb_eval = lgb.Dataset(X_val, y_val, categorical_feature = ['part', 'prior_question_had_explanation_enc'], reference=lgb_train )
val_soces = validation_scores['MSLE'].tolist()
print(f'Mean validation score: {np.average(val_soces):0.5f}' )<save_to_csv>
model = lgb.train(
 params, lgb_train,
valid_sets=[lgb_train, lgb_eval],
verbose_eval=50,
num_boost_round=10000,
early_stopping_rounds=12
)
submit = pd.read_csv('.. /input/covid19-global-forecasting-week-3/submission.csv')
 submit['Fatalities'] = test['Fatalities'].astype('float')
 submit['ConfirmedCases'] = test['ConfirmedCases'].astype('float')
 submit.to_csv('submission.csv',index=False )<set_options>
y_pred = model.predict(X_val)
 y_true = np.array(y_val)
 roc_auc_score(y_true, y_pred )
%matplotlib inline
%config InlineBackend.figure_format = 'retina' 
<load_from_csv>
import matplotlib.pyplot as plt
import seaborn as sns
def get_cpmp_sub(save_oof=False, save_public_test=False):
train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-3/train.csv')
 train['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 train['day'] = train.Date.dt.dayofyear
train['geo'] = ['_'.join(x)for x in zip(train['Country_Region'], train['Province_State'])]
train
test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-3/test.csv')
 test['Province_State'].fillna('', inplace=True)
 test['Date'] = pd.to_datetime(test['Date'])
 test['day'] = test.Date.dt.dayofyear
test['geo'] = ['_'.join(x)for x in zip(test['Country_Region'], test['Province_State'])]
test
day_min = train['day'].min()
train['day'] -= day_min
test['day'] -= day_min
min_test_val_day = test.day.min()
max_test_val_day = train.day.max()
max_test_day = test.day.max()
num_days = max_test_day + 1
min_test_val_day, max_test_val_day, num_days
train['ForecastId'] = -1
test['Id'] = -1
test['ConfirmedCases'] = 0
test['Fatalities'] = 0
debug = False
data = pd.concat([train,
test[test.day > max_test_val_day][train.columns]
] ).reset_index(drop=True)
 if debug:
data = data[data['geo'] >= 'France_'].reset_index(drop=True)
 gc.collect()
dates = data[data['geo'] == 'France_'].Date.values
if 0:
gr = data.groupby('geo')
 data['ConfirmedCases'] = gr.ConfirmedCases.transform('cummax')
 data['Fatalities'] = gr.Fatalities.transform('cummax')
 geo_data = data.pivot(index='geo', columns='day', values='ForecastId')
 num_geo = geo_data.shape[0]
geo_data
geo_id = {}
for i,g in enumerate(geo_data.index):
geo_id[g] = i
ConfirmedCases = data.pivot(index='geo', columns='day', values='ConfirmedCases')
 Fatalities = data.pivot(index='geo', columns='day', values='Fatalities')
 if debug:
cases = ConfirmedCases.values
deaths = Fatalities.values
else:
cases = np.log1p(ConfirmedCases.values)
 deaths = np.log1p(Fatalities.values)
 def get_dataset(start_pred, num_train, lag_period):
days = np.arange(start_pred - num_train + 1, start_pred + 1)
 lag_cases = np.vstack([cases[:, d - lag_period : d] for d in days])
 lag_deaths = np.vstack([deaths[:, d - lag_period : d] for d in days])
 target_cases = np.vstack([cases[:, d : d + 1] for d in days])
 target_deaths = np.vstack([deaths[:, d : d + 1] for d in days])
 geo_ids = np.vstack([geo_ids_base for d in days])
 country_ids = np.vstack([country_ids_base for d in days])
 return lag_cases, lag_deaths, target_cases, target_deaths, geo_ids, country_ids, days
def update_valid_dataset(data, pred_death, pred_case):
lag_cases, lag_deaths, target_cases, target_deaths, geo_ids, country_ids, days = data
day = days[-1] + 1
new_lag_cases = np.hstack([lag_cases[:, 1:], pred_case])
 new_lag_deaths = np.hstack([lag_deaths[:, 1:], pred_death])
 new_target_cases = cases[:, day:day+1]
new_target_deaths = deaths[:, day:day+1]
new_geo_ids = geo_ids
new_country_ids = country_ids
new_days = 1 + days
return new_lag_cases, new_lag_deaths, new_target_cases, new_target_deaths, new_geo_ids, new_country_ids, new_days
def fit_eval(lr_death, lr_case, data, start_lag_death, end_lag_death, num_lag_case, fit, score):
lag_cases, lag_deaths, target_cases, target_deaths, geo_ids, country_ids, days = data
X_death = np.hstack([lag_cases[:, -start_lag_death:-end_lag_death], country_ids])
 X_death = np.hstack([lag_deaths[:, -num_lag_case:], country_ids])
 X_death = np.hstack([lag_cases[:, -start_lag_death:-end_lag_death], lag_deaths[:, -num_lag_case:], country_ids])
 y_death = target_deaths
y_death_prev = lag_deaths[:, -1:]
if fit:
if 0:
keep =(y_death > 0 ).ravel()
X_death = X_death[keep]
y_death = y_death[keep]
y_death_prev = y_death_prev[keep]
lr_death.fit(X_death, y_death)
 y_pred_death = lr_death.predict(X_death)
 y_pred_death = np.maximum(y_pred_death, y_death_prev)
 X_case = np.hstack([lag_cases[:, -num_lag_case:], geo_ids])
 X_case = lag_cases[:, -num_lag_case:]
y_case = target_cases
y_case_prev = lag_cases[:, -1:]
if fit:
lr_case.fit(X_case, y_case)
 y_pred_case = lr_case.predict(X_case)
 y_pred_case = np.maximum(y_pred_case, y_case_prev)
 if score:
death_score = val_score(y_death, y_pred_death)
 case_score = val_score(y_case, y_pred_case)
 else:
death_score = 0
case_score = 0
return death_score, case_score, y_pred_death, y_pred_case
def train_model(train, valid, start_lag_death, end_lag_death, num_lag_case, num_val, score=True):
alpha = 3
lr_death = Ridge(alpha=alpha, fit_intercept=False)
 lr_case = Ridge(alpha=alpha, fit_intercept=True)
(train_death_score, train_case_score, train_pred_death, train_pred_case, 
)= fit_eval(lr_death, lr_case, train, start_lag_death, end_lag_death, num_lag_case, fit=True, score=score)
 death_scores = []
case_scores = []
death_pred = []
case_pred = []
for i in range(num_val): 
(valid_death_score, valid_case_score, valid_pred_death, valid_pred_case, 
)= fit_eval(lr_death, lr_case, valid, start_lag_death, end_lag_death, num_lag_case, fit=False, score=score)
 death_scores.append(valid_death_score)
 case_scores.append(valid_case_score)
 death_pred.append(valid_pred_death)
 case_pred.append(valid_pred_case)
 if 0:
print('val death: %0.3f' % valid_death_score,
'val case: %0.3f' % valid_case_score,
'val : %0.3f' % np.mean([valid_death_score, valid_case_score]),
flush=True)
 valid = update_valid_dataset(valid, valid_pred_death, valid_pred_case)
 if score:
death_scores = np.sqrt(np.mean([s**2 for s in death_scores]))
case_scores = np.sqrt(np.mean([s**2 for s in case_scores]))
if 0:
print('train death: %0.3f' % train_death_score,
'train case: %0.3f' % train_case_score,
'val death: %0.3f' % death_scores,
'val case: %0.3f' % case_scores,
'val : %0.3f' %(( death_scores + case_scores)/ 2),
flush=True)
 else:
print('%0.4f' % case_scores,
', %0.4f' % death_scores,
'= %0.4f' %(( death_scores + case_scores)/ 2),
flush=True)
 death_pred = np.hstack(death_pred)
 case_pred = np.hstack(case_pred)
 return death_scores, case_scores, death_pred, case_pred
countries = [g.split('_')[0] for g in geo_data.index]
countries = pd.factorize(countries)[0]
country_ids_base = countries.reshape(( -1, 1))
ohe = OneHotEncoder(sparse=False)
 country_ids_base = 0.2 * ohe.fit_transform(country_ids_base)
 country_ids_base.shape
geo_ids_base = np.arange(num_geo ).reshape(( -1, 1))
ohe = OneHotEncoder(sparse=False)
 geo_ids_base = 0.1 * ohe.fit_transform(geo_ids_base)
 geo_ids_base.shape
def val_score(true, pred):
pred = np.log1p(np.round(np.expm1(pred)- 0.2))
return np.sqrt(mean_squared_error(true.ravel() , pred.ravel()))
def val_score(true, pred):
return np.sqrt(mean_squared_error(true.ravel() , pred.ravel()))
start_lag_death, end_lag_death = 14, 6,
num_train = 5
num_lag_case = 14
lag_period = max(start_lag_death, num_lag_case)
 def get_oof(start_val_delta=0):
start_val = min_test_val_day + start_val_delta
last_train = start_val - 1
num_val = max_test_val_day - start_val + 1
print(dates[start_val], start_val, num_val)
 train_data = get_dataset(last_train, num_train, lag_period)
 valid_data = get_dataset(start_val, 1, lag_period)
 _, _, val_death_preds, val_case_preds = train_model(train_data, valid_data,
start_lag_death, end_lag_death, num_lag_case, num_val)
 pred_deaths = Fatalities.iloc[:, start_val:start_val+num_val].copy()
pred_deaths.iloc[:, :] = np.expm1(val_death_preds)
 pred_deaths = pred_deaths.stack().reset_index()
pred_deaths.columns = ['geo', 'day', 'Fatalities']
pred_deaths
pred_cases = ConfirmedCases.iloc[:, start_val:start_val+num_val].copy()
pred_cases.iloc[:, :] = np.expm1(val_case_preds)
 pred_cases = pred_cases.stack().reset_index()
pred_cases.columns = ['geo', 'day', 'ConfirmedCases']
pred_cases
sub = train[['Date', 'Id', 'geo', 'day']]
sub = sub.merge(pred_cases, how='left', on=['geo', 'day'])
 sub = sub.merge(pred_deaths, how='left', on=['geo', 'day'])
 sub = sub[sub.day >= start_val]
sub = sub[['Id', 'ConfirmedCases', 'Fatalities']].copy()
return sub
if save_oof:
for start_val_delta, date in zip(range(3, -8, -3),
['2020-03-22', '2020-03-19', '2020-03-16', '2020-03-13']):
print(date, end=' ')
 oof = get_oof(start_val_delta)
 oof.to_csv('.. /submissions/cpmp-%s.csv' % date, index=None)
 def get_sub(start_val_delta=0):
start_val = min_test_val_day + start_val_delta
last_train = start_val - 1
num_val = max_test_val_day - start_val + 1
print(dates[last_train], start_val, num_val)
 num_lag_case = 14
train_data = get_dataset(last_train, num_train, lag_period)
 valid_data = get_dataset(start_val, 1, lag_period)
 _, _, val_death_preds, val_case_preds = train_model(train_data, valid_data,
start_lag_death, end_lag_death, num_lag_case, num_val)
 pred_deaths = Fatalities.iloc[:, start_val:start_val+num_val].copy()
pred_deaths.iloc[:, :] = np.expm1(val_death_preds)
 pred_deaths = pred_deaths.stack().reset_index()
pred_deaths.columns = ['geo', 'day', 'Fatalities']
pred_deaths
pred_cases = ConfirmedCases.iloc[:, start_val:start_val+num_val].copy()
pred_cases.iloc[:, :] = np.expm1(val_case_preds)
 pred_cases = pred_cases.stack().reset_index()
pred_cases.columns = ['geo', 'day', 'ConfirmedCases']
pred_cases
sub = test[['Date', 'ForecastId', 'geo', 'day']]
sub = sub.merge(pred_cases, how='left', on=['geo', 'day'])
 sub = sub.merge(pred_deaths, how='left', on=['geo', 'day'])
 sub = sub.fillna(0)
 sub = sub[['ForecastId', 'ConfirmedCases', 'Fatalities']]
return sub
return sub
known_test = train[['geo', 'day', 'ConfirmedCases', 'Fatalities']
].merge(test[['geo', 'day', 'ForecastId']], how='left', on=['geo', 'day'])
 known_test = known_test[['ForecastId', 'ConfirmedCases', 'Fatalities']][known_test.ForecastId.notnull() ].copy()
known_test
unknow_test = test[test.day > max_test_val_day]
unknow_test
def get_final_sub() :
start_val = max_test_val_day + 1
last_train = start_val - 1
num_val = max_test_day - start_val + 1
print(dates[last_train], start_val, num_val)
 num_lag_case = num_val + 3
train_data = get_dataset(last_train, num_train, lag_period)
 valid_data = get_dataset(start_val, 1, lag_period)
(_, _, val_death_preds, val_case_preds 
)= train_model(train_data, valid_data, start_lag_death, end_lag_death, num_lag_case, num_val, score=False)
 pred_deaths = Fatalities.iloc[:, start_val:start_val+num_val].copy()
pred_deaths.iloc[:, :] = np.expm1(val_death_preds)
 pred_deaths = pred_deaths.stack().reset_index()
pred_deaths.columns = ['geo', 'day', 'Fatalities']
pred_deaths
pred_cases = ConfirmedCases.iloc[:, start_val:start_val+num_val].copy()
pred_cases.iloc[:, :] = np.expm1(val_case_preds)
 pred_cases = pred_cases.stack().reset_index()
pred_cases.columns = ['geo', 'day', 'ConfirmedCases']
pred_cases
print(unknow_test.shape, pred_deaths.shape, pred_cases.shape)
 sub = unknow_test[['Date', 'ForecastId', 'geo', 'day']]
sub = sub.merge(pred_cases, how='left', on=['geo', 'day'])
 sub = sub.merge(pred_deaths, how='left', on=['geo', 'day'])
 sub = sub[['ForecastId', 'ConfirmedCases', 'Fatalities']]
sub = pd.concat([known_test, sub])
 return sub
if save_public_test:
sub = get_sub()
else:
sub = get_final_sub()
return sub<load_from_csv>
iter_test = env.iter_test()
<data_type_conversions><EOS> 
for(test_df, sample_prediction_df)in iter_test:
test_df = pd.merge(test_df, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left')
 test_df = pd.merge(test_df, results_u_final, on=['user_id'], how="left")
 test_df = pd.merge(test_df, results_c_final, on=['content_id'], how="left")
 test_df['answered_correctly_user'].fillna(0.5, inplace=True)
 test_df['answered_correctly_content'].fillna(0.5, inplace=True)
 test_df['part'] = test_df.part - 1
test_df['part'].fillna(4, inplace = True)
 test_df['sum'].fillna(0, inplace=True)
 test_df['count'].fillna(0, inplace=True)
 test_df['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df["prior_question_had_explanation_enc"] = lb_make.fit_transform(test_df["prior_question_had_explanation"])
 test_df['answered_correctly'] = model.predict(test_df[['answered_correctly_user', 'answered_correctly_content', 'sum', 'count',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part']])
 env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )