Kaggle competition: COVID19 Global Forecasting (Week 4)
sub_blend=sub.copy()
sub_blend['target']=preds_blend
sub_blend.to_csv(OUT_FOLDER/'submission_blend.csv', index=False )<compute_test_metric>
plt.style.use('fivethirtyeight')
 le = preprocessing.LabelEncoder()
warnings.filterwarnings('ignore' )
corr=np.corrcoef(rankdata(sub_v9)/len(sub_v9),
rankdata(sub['target'].values)/len(sub))
print(f"The correlation with the benchmark submission:
{corr}." )<set_options>
import plotly.io as pio
import plotly.express as px
import plotly.graph_objects as go
del auc
gc.collect()<import_modules>
train_df = pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 test_df = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 submission = pd.read_csv(".. /input/covid19-global-forecasting-week-4/submission.csv" )
from sklearn.metrics import roc_curve, auc<set_options>
display(train_df.isnull().sum() /len(train_df)*100)
 display(test_df.isnull().sum() /len(test_df)*100 )
warnings.filterwarnings('ignore')
<load_from_csv>
train_df.rename(columns={'Province_State':'State','Country_Region':'Country'}, inplace=True)
 test_df.rename(columns={'Province_State':'State','Country_Region':'Country'}, inplace=True )
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 seed_everything(42)
 def read_data() :
train = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/train.csv')
 test = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')
 tr_ef1 = pd.read_csv('/kaggle/input/eff-b3/EfficientNetB3_256.csv')
 te_ef1 = pd.read_csv('/kaggle/input/eff-b3/sub_EfficientNetB3_256.csv')
 tr_ef2 = pd.read_csv('/kaggle/input/effb3-512/EfficientNetB3_512.csv')
 te_ef2 = pd.read_csv('/kaggle/input/effb3-512/sub_EfficientNetB3_512.csv')
 tr_ef3 = pd.read_csv('/kaggle/input/effb3-384/EfficientNetB3_384.csv')
 te_ef3 = pd.read_csv('/kaggle/input/effb3-384/sub_EfficientNetB3_384.csv')
 tr_ef4 = pd.read_csv('/kaggle/input/eff-256v2/EfficientNetB0_256.csv')
 te_ef4 = pd.read_csv('/kaggle/input/eff-256v2/sub_EfficientNetB0_256.csv')
 roc_auc_256 = metrics.roc_auc_score(tr_ef1['target'], tr_ef1['predictions'])
 roc_auc_512 = metrics.roc_auc_score(tr_ef2['target'], tr_ef2['predictions'])
 roc_auc_384 = metrics.roc_auc_score(tr_ef3['target'], tr_ef3['predictions'])
 roc_auc_256_B0 = metrics.roc_auc_score(tr_ef4['target'], tr_ef4['predictions'])
 print(f'EfficientnetB3 model for 256 x 256 images out of folds roc auc score is {roc_auc_256}')
 print(f'EfficientnetB3 model for 512 x 512 images out of folds roc auc score is {roc_auc_512}')
 print(f'EfficientnetB3 model for 384 x 384 images out of folds roc auc score is {roc_auc_384}, this model includes meta data')
 print(f'EfficientnetB0 model for 256 x 256 images out of folds roc auc score is {roc_auc_256_B0}, this model includes meta data')
 te_ef1.columns = ['image_name', 'predictions']
tr_ef1 = tr_ef1[['image_name', 'predictions']]
te_ef2.columns = ['image_name', 'predictions_1']
tr_ef2 = tr_ef2[['image_name', 'predictions']]
tr_ef2.columns = ['image_name', 'predictions_1']
te_ef3.columns = ['image_name', 'predictions_2']
tr_ef3 = tr_ef3[['image_name', 'predictions']]
tr_ef3.columns = ['image_name', 'predictions_2']
te_ef4.columns = ['image_name', 'predictions_3']
tr_ef4 = tr_ef4[['image_name', 'predictions']]
tr_ef4.columns = ['image_name', 'predictions_3']
train = train.merge(tr_ef1, on = 'image_name' ).merge(tr_ef2, on = 'image_name' ).merge(tr_ef3, on = 'image_name' ).merge(tr_ef4, on = 'image_name')
 test = test.merge(te_ef1, on = 'image_name' ).merge(te_ef2, on = 'image_name' ).merge(te_ef3, on = 'image_name' ).merge(te_ef4, on = 'image_name')
 return train, test, sub
def feature_engineering(train, test):
trn_images = train['image_name'].values
trn_sizes = np.zeros(( trn_images.shape[0], 2))
for i, img_path in enumerate(tqdm(trn_images)) :
img = Image.open(os.path.join('/kaggle/input/siim-isic-melanoma-classification/jpeg/train/', f'{img_path}.jpg'))
trn_sizes[i] = np.array([img.size[0], img.size[1]])
 test_images = test['image_name'].values
test_sizes = np.zeros(( test_images.shape[0],2))
for i, img_path in enumerate(tqdm(test_images)) :
img = Image.open(os.path.join('/kaggle/input/siim-isic-melanoma-classification/jpeg/test/', f'{img_path}.jpg'))
test_sizes[i] = np.array([img.size[0],img.size[1]])
 train['w'] = trn_sizes[:,0]
train['h'] = trn_sizes[:,1]
test['w'] = test_sizes[:,0]
test['h'] = test_sizes[:,1]
return train, test
def encode_categorical(train, test):
for col in ['sex', 'anatom_site_general_challenge']:
encoder = preprocessing.LabelEncoder()
train[col].fillna('unknown', inplace = True)
 test[col].fillna('unknown', inplace = True)
 train[col] = encoder.fit_transform(train[col])
 test[col] = encoder.transform(test[col])
 age_approx = np.nanmean(np.concatenate([np.array(train['age_approx']), np.array(test['age_approx'])]))
train['age_approx'].fillna(age_approx, inplace = True)
 test['age_approx'].fillna(age_approx, inplace = True)
 train['patient_id'].fillna('unknown', inplace = True)
 return train, test
def train_and_evaluate_cat(train, test, cat_params, verbose_eval, folds = 5):
features = [col for col in train.columns if col not in ['image_name', 'patient_id', 'diagnosis', 'benign_malignant', 'target', 'source']]
if verbose_eval != False:
print('Training with features: ', features)
 kf = GroupKFold(n_splits = folds)
 target = 'target'
oof_pred = np.zeros(len(train))
y_pred = np.zeros(len(test))
for fold,(tr_ind, val_ind)in enumerate(kf.split(train, groups = train['patient_id'])) :
if verbose_eval != False:
print('
')
 print('-'*50)
 print(f'Training fold {fold + 1}"')
 x_train, x_val = train[features].iloc[tr_ind], train[features].iloc[val_ind]
y_train, y_val = train[target][tr_ind], train[target][val_ind]
model = CatBoostClassifier(**cat_params)
 model.fit(x_train, y_train, eval_set =(x_val, y_val), cat_features = ['sex', 'anatom_site_general_challenge'],
use_best_model = True, early_stopping_rounds = 50, verbose_eval = verbose_eval)
 oof_pred[val_ind] = model.predict_proba(x_val)[:, 1]
y_pred += model.predict_proba(test[features])[:, 1] / kf.n_splits
rauc = metrics.roc_auc_score(train['target'], oof_pred)
 if verbose_eval != False:
print(f'Our oof roc auc score for our cat boost model is {rauc}')
 gc.collect()
return rauc, y_pred
train, test, sub = read_data()
train, test = feature_engineering(train, test)
 train, test = encode_categorical(train, test )<train_model>
def missings(state, country):
return country if pd.isna(state)== True else state
def run_cat_bayesian(learning_rate, depth, bagging_temperature, colsample_bylevel):
params = {
'learning_rate': learning_rate,
'eval_metric': 'AUC',
'loss_function': 'Logloss',
'random_seed': 42,
'task_type': 'CPU',
'depth': int(depth),
'bagging_temperature': bagging_temperature,
'colsample_bylevel': colsample_bylevel,
}
roc_auc, y_pred = train_and_evaluate_cat(train, test, params, False)
 return roc_auc
bounds_cat = {
'learning_rate':(0.1, 0.5),
'depth':(2, 12),
'bagging_temperature':(0.0, 2.0),
'colsample_bylevel':(0.5, 1.0)
 }
cat_bo = BayesianOptimization(run_cat_bayesian, bounds_cat, random_state = 42)
 cat_bo.maximize(init_points = 100, n_iter = 100, acq = 'ucb', xi = 0.0, alpha = 1e-6)
 params = {
'learning_rate': cat_bo.max['params']['learning_rate'],
'eval_metric': 'AUC',
'loss_function': 'Logloss',
'random_seed': 42,
'task_type': 'CPU',
'depth': int(cat_bo.max['params']['depth']),
'bagging_temperature': cat_bo.max['params']['bagging_temperature'],
'colsample_bylevel': cat_bo.max['params']['colsample_bylevel']
}
roc_auc, y_pred = train_and_evaluate_cat(train, test, params, 50)
 test['target'] = y_pred
sub = test[['image_name', 'target']]
sub.to_csv('cat_baseline_sub.csv', index = False )<load_from_csv>
train_df['State'] = train_df.apply(lambda x: missings(x['State'],x['Country']),axis=1)
 test_df['State'] = test_df.apply(lambda x: missings(x['State'],x['Country']),axis=1 )
env = riiideducation.make_env()
train= pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',
usecols=[1, 2, 3,4,7,8,9], dtype={'timestamp': 'int64', 'user_id': 'int32' ,'content_id': 'int16','content_type_id': 'int8','answered_correctly':'int8','prior_question_elapsed_time': 'float32','prior_question_had_explanation': 'boolean'} 
)
 train = train[train.content_type_id == False]
train = train.sort_values(['timestamp'], ascending=True)
 train.drop(['timestamp','content_type_id'], axis=1, inplace=True)
 results_c = train[['content_id','answered_correctly']].groupby(['content_id'] ).agg(['mean'])
 results_c.columns = ["answered_correctly_content"]
results_u = train[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean', 'sum'])
 results_u.columns = ["answered_correctly_user", 'sum']<load_from_csv>
print("In our data set are {} countries and {} states.".format(train_df['Country'].nunique() ,train_df['State'].nunique()))
questions_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv',
usecols=[0,1, 3,4],
dtype={'question_id': 'int16',
'part': 'int8','bundle_id': 'int8','tags': 'str'} 
)
 tag = questions_df["tags"].str.split(" ", n = 10, expand = True)
 tag.columns = ['tags1','tags2','tags3','tags4','tags5','tags6']
questions_df = pd.concat([questions_df,tag],axis=1)
 questions_df['tags1'] = pd.to_numeric(questions_df['tags1'], errors='coerce')
 questions_df['tags2'] = pd.to_numeric(questions_df['tags2'], errors='coerce')
 questions_df['tags3'] = pd.to_numeric(questions_df['tags3'], errors='coerce')
 questions_df['tags4'] = pd.to_numeric(questions_df['tags4'], errors='coerce')
 questions_df['tags5'] = pd.to_numeric(questions_df['tags5'], errors='coerce')
 questions_df['tags6'] = pd.to_numeric(questions_df['tags6'], errors='coerce' )<merge>
df_plot = train_df.loc[: , ['Date', 'Country', 'ConfirmedCases', 'Fatalities']].groupby(['Date', 'Country'] ).max().reset_index()
df_plot.loc[:, 'Size'] = np.power(df_plot["ConfirmedCases"]+1,0.3)-1
fig = px.scatter_geo(df_plot,
locations="Country",
locationmode = "country names",
hover_name="Country",
color="ConfirmedCases",
animation_frame="Date",
size='Size',
projection="natural earth",
title="Global Spread of Covid-19",
width=1500, height=800)
 fig.layout.updatemenus[0].buttons[0].args[1]["frame"]["duration"] = 5
fig.show()
X = train.iloc[80000000:,:]
X['prior_question_had_explanation'].fillna(False, inplace=True)
 X = pd.merge(X, results_u, on=['user_id'], how="left")
 X = pd.merge(X, results_c, on=['content_id'], how="left")
 X = pd.merge(X, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left')
 X=X[X.answered_correctly!= -1 ]
X=X.sort_values(['user_id'])
 Y = X[["answered_correctly"]]
X = X.drop(["answered_correctly"], axis=1 )<categorify>
data_leak = pd.merge(train_df,test_df, how='inner', on='Date')['Date'].unique().tolist()
data_leak.append('2020-04-01')
 data_leak.sort()
print("Both data sets contain the following dates: {}".format(data_leak))
lb_make = LabelEncoder()
X["prior_question_had_explanation_enc"] = lb_make.fit_transform(X["prior_question_had_explanation"])
 X.head()
X = X[['answered_correctly_user', 'answered_correctly_content', 'sum','bundle_id','part','prior_question_elapsed_time','prior_question_had_explanation_enc','tags1','tags2','tags3']]
X.fillna(0.5, inplace=True)
 Xt, Xv, Yt, Yv = train_test_split(X, Y, test_size =0.2, shuffle=False)
 params = {
'objective': 'binary',
'max_bin': 600,
'learning_rate': 0.02,
'num_leaves': 80
}
lgb_train = lgb.Dataset(Xt, Yt,categorical_feature = ['part','tags1','tags2','tags3'])
 lgb_eval = lgb.Dataset(Xv, Yv, reference=lgb_train,categorical_feature = ['part','tags1','tags2','tags3'])
 model = lgb.train(
 params, lgb_train,
valid_sets=[lgb_train, lgb_eval],
verbose_eval=10,
num_boost_round=10000,
early_stopping_rounds=10
)<compute_train_metric>
def create_features(df):
df['Day_num'] = le.fit_transform(df['Date'])
 df['Date'] = pd.to_datetime(df['Date'])
 df['Day'] = df['Date'].dt.day
df['Week'] = df['Date'].dt.week
df['Month'] = df['Date'].dt.month
df['DayOfWeek'] = df['Date'].dt.dayofweek
df['Country'] = le.fit_transform(df['Country'])
 country_dict = dict(zip(le.inverse_transform(df['Country']), df['Country']))
df['State'] = le.fit_transform(df['State'])
 state_dict = dict(zip(le.inverse_transform(df['State']), df['State']))
return df, country_dict, state_dict
y_pred = model.predict(Xv)
 y_true = np.array(Yv)
 roc_auc_score(y_true, y_pred )<merge>
df_all, country_dict, state_dict = create_features(df_all )
iter_test = env.iter_test()
for(test_df, sample_prediction_df)in iter_test:
test_df = test_df.sort_values(['user_id','timestamp'], ascending=False)
 test_df['answer_time'] = test_df.groupby(['user_id'])['prior_question_elapsed_time'].shift(1)
 test_df = pd.merge(test_df, results_u, on=['user_id'], how="left")
 test_df = pd.merge(test_df, results_c, on=['content_id'], how="left")
 test_df = pd.merge(test_df, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left')
 test_df['answered_correctly_user'].fillna(0.5, inplace=True)
 test_df['answered_correctly_content'].fillna(0.5, inplace=True)
 test_df['sum'].fillna(0, inplace=True)
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df["prior_question_had_explanation_enc"] = lb_make.fit_transform(test_df["prior_question_had_explanation"])
 test_df['answered_correctly'] = model.predict(test_df[['answered_correctly_user', 'answered_correctly_content', 'sum','bundle_id','part','prior_question_elapsed_time','prior_question_had_explanation_enc',
'tags1','tags2','tags3']])
 env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )<install_modules>
def train_test_split_extend(df,d,day,filter_col_confirmed,filter_col_fatalities):
df=df.loc[df['Day_num'] >= day]
df_train = df.loc[df['Day_num'] < d]
X_train = df_train
Y_train_1 = df_train['ConfirmedCases']
Y_train_2 = df_train['Fatalities']
X_train_1 = X_train.drop(columns=filter_col_fatalities ).drop(columns='ConfirmedCases')
 X_train_2 = X_train.drop(columns=filter_col_confirmed ).drop(columns='Fatalities')
 df_test = df.loc[df['Day_num'] == d]
x_test = df_test
x_test_1 = x_test.drop(columns=filter_col_fatalities ).drop(columns='ConfirmedCases')
 x_test_2 = x_test.drop(columns=filter_col_confirmed ).drop(columns='Fatalities')
 x_test.drop(['ConfirmedCases', 'Fatalities'], axis=1, inplace=True)
 X_train_1.drop('Id', inplace=True, errors='ignore', axis=1)
 X_train_1.drop('ForecastId', inplace=True, errors='ignore', axis=1)
 X_train_2.drop('Id', inplace=True, errors='ignore', axis=1)
 X_train_2.drop('ForecastId', inplace=True, errors='ignore', axis=1)
 x_test_1.drop('Id', inplace=True, errors='ignore', axis=1)
 x_test_1.drop('ForecastId', inplace=True, errors='ignore', axis=1)
 x_test_2.drop('Id', inplace=True, errors='ignore', axis=1)
 x_test_2.drop('ForecastId', inplace=True, errors='ignore', axis=1)
 return X_train_1, X_train_2, Y_train_1, Y_train_2, x_test_1, x_test_2
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1<set_options>
def lin_reg(X_train, Y_train, x_test):
regr = linear_model.LinearRegression()
regr.fit(X_train, Y_train)
 pred = regr.predict(x_test)
 return regr, pred
pd.set_option('display.max_columns', 50)
<define_variables>
def country_calculation(df_all,country,date,day):
df_country = df_all.copy()
df_country = df_country.loc[df_country['Date'] >= date]
df_country = df_country.loc[df_country['Country'] == country_dict[country]]
features = ['Id', 'State', 'Country','ConfirmedCases', 'Fatalities', 'Day_num']
df_country = df_country[features]
df_country = lag_feature(df_country, 'ConfirmedCases',range(1, 40))
df_country = lag_feature(df_country, 'Fatalities', range(1,20))
filter_col_confirmed = [col for col in df_country if col.startswith('Confirmed')]
filter_col_fatalities= [col for col in df_country if col.startswith('Fataliti')]
filter_col = np.append(filter_col_confirmed, filter_col_fatalities)
 df_country[filter_col] = df_country[filter_col].apply(lambda x: np.log1p(x))
df_country.replace([np.inf, -np.inf], 0, inplace=True)
 df_country.fillna(0, inplace=True)
 start = df_country[df_country['Id']==-999].Day_num.min()
end = df_country[df_country['Id']==-999].Day_num.max()
for d in range(start,end+1):
X_train_1, X_train_2, Y_train_1, Y_train_2, x_test_1, x_test_2 = train_test_split_extend(df_country,d,day,filter_col_confirmed,filter_col_fatalities)
 regr_1, pred_1 = lin_reg(X_train_1, Y_train_1, x_test_1)
 df_country.loc[(df_country['Day_num'] == d)&(df_country['Country'] == country_dict[country]), 'ConfirmedCases'] = pred_1[0]
regr_2, pred_2 = lin_reg(X_train_2, Y_train_2, x_test_2)
 df_country.loc[(df_country['Day_num'] == d)&(df_country['Country'] == country_dict[country]), 'Fatalities'] = pred_2[0]
df_country = lag_feature(df_country, 'ConfirmedCases',range(1, 40))
df_country = lag_feature(df_country, 'Fatalities', range(1,20))
df_country.replace([np.inf, -np.inf], 0, inplace=True)
 df_country.fillna(0, inplace=True)
 print("Calculation done.")
 return df_country
data_types_dict = {
'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id':'int8',
'task_container_id': 'int16',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'bool'
}
target = 'answered_correctly'
features_dict = {
'user_interaction_count':'int16',
'lagtime':'float32',
'lagtime2':'float32',
'lagtime3':'float32',
'content_id':'int16',
'task_container_id':'int16',
'user_lecture_lv':'float16',
'prior_question_elapsed_time':'float32',
'user_correctness':'float16',
'user_uncorrect_count':'int16',
'user_correct_count':'int16',
'content_correctness_std':'float16',
'content_correct_count':'int32',
'content_uncorrect_count':'int32',
'content_had_explanation_mean':'float16',
'content_explation_false_mean':'float16',
'content_explation_true_mean':'float16',
'task_container_cor_count':'int32',
'attempt_no':'int8',
'part':'int8',
'part_correctness_std':'float16',
'tags1': 'int16',
'tags2': 'int16',
'bundle_id':'int16',
'part_bundle_id':'int32',
'prior_question_had_explanation':'int8',
'explanation_mean':'float16',
'explanation_false_count':'int16',
'explanation_true_count':'int16',
}<load_from_csv>
def country_state_calculation(df_all,country, state, date,day):
df_country = df_all.copy()
df_country = df_country.loc[df_country['Date'] >= date]
df_country = df_country.loc[df_country['Country'] == country_dict[country] &(df_country['State']==state_dict[state])]
features = ['Id', 'State', 'Country','ConfirmedCases', 'Fatalities', 'Day_num']
df_country = df_country[features]
df_country = lag_feature(df_country, 'ConfirmedCases',range(1, 40))
df_country = lag_feature(df_country, 'Fatalities', range(1,20))
filter_col_confirmed = [col for col in df_country if col.startswith('Confirmed')]
filter_col_fatalities= [col for col in df_country if col.startswith('Fataliti')]
filter_col = np.append(filter_col_confirmed, filter_col_fatalities)
 df_country[filter_col] = df_country[filter_col].apply(lambda x: np.log1p(x))
df_country.replace([np.inf, -np.inf], 0, inplace=True)
 df_country.fillna(0, inplace=True)
 start = df_country[df_country['Id']==-999].Day_num.min()
end = df_country[df_country['Id']==-999].Day_num.max()
for d in range(start,end+1):
X_train_1, X_train_2, Y_train_1, Y_train_2, x_test_1, x_test_2 = train_test_split_extend(df_country,d,day,filter_col_confirmed,filter_col_fatalities)
 regr_1, pred_1 = lin_reg(X_train_1, Y_train_1, x_test_1)
 df_country.loc[(df_country['Day_num'] == d)&(df_country['Country'] == country_dict[country])&(df_country['State'] == state_dict[state]), 'ConfirmedCases'] = pred_1[0]
regr_2, pred_2 = lin_reg(X_train_2, Y_train_2, x_test_2)
 df_country.loc[(df_country['Day_num'] == d)&(df_country['Country'] == country_dict[country]), 'Fatalities'] = pred_2[0]
df_country = lag_feature(df_country, 'ConfirmedCases',range(1, 10))
df_country = lag_feature(df_country, 'Fatalities', range(1,8))
df_country.replace([np.inf, -np.inf], 0, inplace=True)
 df_country.fillna(0, inplace=True)
 print("Calculation done.")
 return df_country
train_df = dt.fread('.. /input/riiid-test-answer-prediction/train.csv', columns=set(data_types_dict.keys())).to_pandas()<data_type_conversions>
user_lecture_agg = train_df.groupby('user_id')['content_type_id'].agg(['sum', 'count'])
 user_lecture_agg = user_lecture_agg.astype('int16')
 cum = train_df.groupby('user_id')['content_type_id'].agg(['cumsum', 'cumcount'])
 cum['cumcount'] = cum['cumcount'] + 1
train_df['user_interaction_count'] = cum['cumcount']
train_df['user_lecture_lv'] = cum['cumsum'] / cum['cumcount']
train_df.user_lecture_lv = train_df.user_lecture_lv.astype('float16')
 train_df.user_interaction_count = train_df.user_interaction_count.astype('int16')
 del cum
gc.collect()<data_type_conversions>
!pip install pyramid.arima
train_df['prior_question_had_explanation'].fillna(False, inplace=True)
 train_df = train_df.astype(data_types_dict)
 train_df = train_df[train_df[target] != -1].reset_index(drop=True )<data_type_conversions>
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)))
 pd.set_option('mode.chained_assignment', None)
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 train = pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 train['Province_State'].fillna('', inplace=True)
 test['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'])
 train = train.sort_values(['Country_Region','Province_State','Date'])
 test = test.sort_values(['Country_Region','Province_State','Date'] )
content_explation_agg=train_df[["content_id","prior_question_had_explanation",target]].groupby(["content_id","prior_question_had_explanation"])[target].agg(['mean'])
 content_explation_agg=content_explation_agg.unstack()
content_explation_agg=content_explation_agg.reset_index()
content_explation_agg.columns = ['content_id', 'content_explation_false_mean','content_explation_true_mean']
content_explation_agg.content_id=content_explation_agg.content_id.astype('int16')
 content_explation_agg.content_explation_false_mean=content_explation_agg.content_explation_false_mean.astype('float16')
 content_explation_agg.content_explation_true_mean=content_explation_agg.content_explation_true_mean.astype('float16' )<data_type_conversions>
feature_day = [1,5,10,15,20,30,40,50,75,100,150,200,300,400,500,750,1000,1250,1500,1750,2000,2250,2500,2750,3000,3250,3500,3750,4000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
train_df["attempt_no"] = 1
train_df.attempt_no=train_df.attempt_no.astype('int8')
 attempt_no_agg=train_df.groupby(["user_id","content_id"])["attempt_no"].agg(['sum'] ).astype('int8')
 train_df["attempt_no"] = train_df[["user_id","content_id",'attempt_no']].groupby(["user_id","content_id"])["attempt_no"].cumsum()
attempt_no_agg=attempt_no_agg[attempt_no_agg['sum'] >1]<data_type_conversions>
pred_data_all = pd.DataFrame()
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
print(country + ' and ' + province)
 df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 for day in sorted(feature_day,reverse = True):
feature_use = 'Number day from ' + str(day)+ ' case'
idx = X_train[X_train[feature_use] == 0].shape[0]
if(X_train[X_train[feature_use] > 0].shape[0] >= 20):
break
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
idx = X_pred[X_pred[feature_use] == 0].shape[0]
adjusted_X_pred = X_pred[idx:][feature_use].values.reshape(-1, 1)
 pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
max_train_date = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].max()
min_test_date = pred_data['Date'].min()
if len(adjusted_y_train_confirmed)< 1:
adjusted_y_train_confirmed = np.zeros(3)
 else:
if len(adjusted_y_train_confirmed)< 2:
adjusted_y_train_confirmed = np.append(adjusted_y_train_confirmed,adjusted_y_train_confirmed[len(adjusted_y_train_confirmed)-1],adjusted_y_train_confirmed[len(adjusted_y_train_confirmed)-1])
 else:
if len(adjusted_y_train_confirmed)< 3:
adjusted_y_train_confirmed = np.append(adjusted_y_train_confirmed,adjusted_y_train_confirmed[len(adjusted_y_train_confirmed)-1])
 else:
pass
model = SARIMAX(adjusted_y_train_confirmed, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_confirmed = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_confirmed = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['ConfirmedCases'].values
y_hat_confirmed = np.concatenate(( y_train_confirmed,y_hat_confirmed), axis = 0)
 if len(adjusted_y_train_fatalities)< 1:
adjusted_y_train_fatalities = np.zeros(3)
 else:
if len(adjusted_y_train_fatalities)< 2:
adjusted_y_train_fatalities = np.append(adjusted_y_train_fatalities,adjusted_y_train_fatalities[len(adjusted_y_train_fatalities)-1],adjusted_y_train_fatalities[len(adjusted_y_train_fatalities)-1])
 else:
if len(adjusted_y_train_fatalities)< 3:
adjusted_y_train_fatalities = np.append(adjusted_y_train_fatalities,adjusted_y_train_fatalities[len(adjusted_y_train_fatalities)-1])
 else:
pass
model = SARIMAX(adjusted_y_train_fatalities, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_fatalities = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_fatalities = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['Fatalities'].values
y_hat_fatalities = np.concatenate(( y_train_fatalities,y_hat_fatalities), axis = 0)
 pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data )
prior_question_elapsed_time_mean=train_df['prior_question_elapsed_time'].mean()
train_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)
 max_timestamp_u = train_df[['user_id','timestamp']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u.columns = ['user_id', 'max_time_stamp']
max_timestamp_u.user_id = max_timestamp_u.user_id.astype('int32')
 train_df['lagtime'] = train_df.groupby('user_id')['timestamp'].shift()
max_timestamp_u2 = train_df[['user_id','lagtime']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u2.columns = ['user_id', 'max_time_stamp2']
max_timestamp_u2.user_id = max_timestamp_u2.user_id.astype('int32')
 train_df['lagtime'] = train_df['timestamp']-train_df['lagtime']
lagtime_mean = train_df['lagtime'].mean()
train_df['lagtime'].fillna(lagtime_mean, inplace=True)
 train_df['lagtime'] = train_df['lagtime']/(1000*3600)
 train_df.lagtime=train_df.lagtime.astype('float32')
 train_df['lagtime2'] = train_df.groupby('user_id')['timestamp'].shift(2)
 max_timestamp_u3 = train_df[['user_id','lagtime2']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u3.columns = ['user_id', 'max_time_stamp3']
max_timestamp_u3.user_id = max_timestamp_u3.user_id.astype('int32')
 train_df['lagtime2'] = train_df['timestamp']-train_df['lagtime2']
lagtime_mean2 = train_df['lagtime2'].mean()
train_df['lagtime2'].fillna(lagtime_mean2, inplace=True)
 train_df['lagtime2'] = train_df['lagtime2']/(1000*3600)
 train_df.lagtime2 = train_df.lagtime2.astype('float32')
 train_df['lagtime3'] = train_df.groupby('user_id')['timestamp'].shift(3)
 train_df['lagtime3'] = train_df['timestamp']-train_df['lagtime3']
lagtime_mean3 = train_df['lagtime3'].mean()
train_df['lagtime3'].fillna(lagtime_mean3, inplace=True)
 train_df['lagtime3'] = train_df['lagtime3']/(1000*3600)
 train_df.lagtime3 = train_df.lagtime3.astype('float32')
 train_df['timestamp'] = train_df['timestamp']/(1000*3600)
 train_df.timestamp = train_df.timestamp.astype('float16' )<data_type_conversions>
df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_3 = df_val.copy()
submission = df_val[['ForecastId','ConfirmedCases_hat','Fatalities_hat']]
submission.columns = ['ForecastId','ConfirmedCases','Fatalities']
submission.to_csv('submission.csv', index=False )
train_df['lag'] = train_df.groupby('user_id')[target].shift()
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 user_agg = train_df.groupby('user_id')['lag'].agg(['sum', 'count'] ).astype('int16')
 cum['cumsum'].fillna(0, inplace=True)
 train_df['user_correctness'] = cum['cumsum'] / cum['cumcount']
train_df['user_correct_count'] = cum['cumsum']
train_df['user_uncorrect_count'] = cum['cumcount']-cum['cumsum']
train_df.drop(columns = ['lag'], inplace=True)
 train_df['user_correctness'].fillna(0.67, inplace=True)
 train_df.user_correctness = train_df.user_correctness.astype('float16')
 train_df.user_correct_count = train_df.user_correct_count.astype('int16')
 train_df.user_uncorrect_count = train_df.user_uncorrect_count.astype('int16')
 del cum
gc.collect()<data_type_conversions>
train_df.prior_question_had_explanation = train_df.prior_question_had_explanation.astype('int8')
 explanation_agg = train_df.groupby('user_id')['prior_question_had_explanation'].agg(['sum', 'count'])
 explanation_agg = explanation_agg.astype('int16')
 cum = train_df.groupby('user_id')['prior_question_had_explanation'].agg(['cumsum', 'cumcount'])
 cum['cumcount'] = cum['cumcount'] + 1
train_df['explanation_mean'] = cum['cumsum'] / cum['cumcount']
train_df['explanation_true_count'] = cum['cumsum']
train_df['explanation_false_count'] = cum['cumcount']-cum['cumsum']
train_df.explanation_mean = train_df.explanation_mean.astype('float16')
 train_df.explanation_true_count = train_df.explanation_true_count.astype('int16')
 train_df.explanation_false_count = train_df.explanation_false_count.astype('int16')
 del cum
gc.collect()<data_type_conversions>
def crosscheck_sarima(country):
crosscheck = train_df[(train_df['Country'] == country)&(train_df['Date'] >= '2020-04-02')].reset_index()
arima = pred_data_all[(pred_data_all['Country_Region'] == country)].reset_index()
arima['ConfirmedCases_In'] = arima['ConfirmedCases_hat']
arima['Fatalities_In'] = arima['Fatalities_hat']
arima['CC_Crosscheck'] = crosscheck['ConfirmedCases']
arima['Fat_Crosscheck'] = crosscheck['Fatalities']
arima['Day_num'] = arima['Date']
return arima
task_container_agg = train_df.groupby('task_container_id')[target].agg(['sum', 'count','var'])
 train_df['task_container_cor_count'] = train_df['task_container_id'].map(task_container_agg['sum'] ).astype('int32')
 task_container_agg = task_container_agg.astype('float32' )<categorify>
def crosscheck_sarima_cs(country,state):
crosscheck = train_df[(train_df['Country'] == country)&(train_df['State'] == state)&(train_df['Date'] >= '2020-04-02')].reset_index()
arima = pred_data_all[(pred_data_all['Country_Region'] == country)&(pred_data_all['Province_State'] == state)].reset_index()
arima['ConfirmedCases_In'] = arima['ConfirmedCases_hat']
arima['Fatalities_In'] = arima['Fatalities_hat']
arima['CC_Crosscheck'] = crosscheck['ConfirmedCases']
arima['Fat_Crosscheck'] = crosscheck['Fatalities']
arima['Day_num'] = arima['Date']
return arima