Kaggle competition: TalkingData Mobile User Demographics
<load_from_csv><EOS> 
%matplotlib inline
<SOS> metric: MulticlassLoss Kaggle data source: talkingdata-mobile-user-demographics<define_variables>
%matplotlib inline
countries_to_replace = [ 
('Czech Republic', 'Czechia'), 
('United States of America', 'US'), 
('Côte d'Ivoire(Ivory Coast)', 'Côte d'Ivoire'), 
('Korea(South)', 'Korea, South'), 
('Swaziland', 'Eswatini'), 
('Gambia', 'The Gambia'), 
('Myanmar(Burma)', 'Myanmar'), 
('East Timor', 'Timor-Leste'), 
('Macedonia', 'North Macedonia'), 
('Cape Verde', 'Cabo Verde'), 
('Congo(Republic)', 'Congo(Brazzaville)'), 
('Congo(Democratic Republic)', 'Congo(Kinshasa)'), 
('Palestinian Territories', 'State of Palestine'), 
('Bahamas', 'The Bahamas'), 
('United Kingdom of Great Britain and Northern Ireland', 'United Kingdom'), 
('Vatican City', 'Holy See')
 ]
csv_dir = base_folder + 'worldbymap/'
files = [
'labor_force',
'death_rate',
'air_traffic_passengers',
'hospital_bed_density',
'obesity',
'old_people',
'physicians_density'
]
wbm = {}
for f in files:
wbm[f] = pd.read_csv(csv_dir + f + '.csv', delimiter=';', decimal=',', na_values='N.A.')
 for ctr in countries_to_replace:
wbm[f] = wbm[f].replace(ctr[0], ctr[1])
 wbm[files[0]]<merge>
datadir = '.. /input'
gatrain = pd.read_csv(os.path.join(datadir,'gender_age_train.csv'),
index_col='device_id')
 gatest = pd.read_csv(os.path.join(datadir,'gender_age_test.csv'),
index_col = 'device_id')
 phone = pd.read_csv(os.path.join(datadir,'phone_brand_device_model.csv'))
phone = phone.drop_duplicates('device_id',keep='first' ).set_index('device_id')
 events = pd.read_csv(os.path.join(datadir,'events.csv'),
parse_dates=['timestamp'], index_col='event_id')
 appevents = pd.read_csv(os.path.join(datadir,'app_events.csv'),
usecols=['event_id','app_id','is_active'],
dtype={'is_active':bool})
 applabels = pd.read_csv(os.path.join(datadir,'app_labels.csv'))
df_add = pd.DataFrame()
for dataset in wbm.keys() :
if df_add.shape ==(0, 0):
df_add = wbm[dataset].copy()
else:
df_add = df_add.merge(wbm[dataset], on='Country', how='left')
 df_add.rename(columns={"Country": "Country/Region"}, inplace=True)
 df_add<merge>
datadir = '.. /input'
gatrain = pd.read_csv(os.path.join(datadir,'gender_age_train.csv'),
index_col='device_id')
 gatest = pd.read_csv(os.path.join(datadir,'gender_age_test.csv'),
index_col = 'device_id')
 phone = pd.read_csv(os.path.join(datadir,'phone_brand_device_model.csv'))
phone = phone.drop_duplicates('device_id',keep='first' ).set_index('device_id')
 events = pd.read_csv(os.path.join(datadir,'events.csv'),
parse_dates=['timestamp'], index_col='event_id')
 appevents = pd.read_csv(os.path.join(datadir,'app_events.csv'),
usecols=['event_id','app_id','is_active'],
dtype={'is_active':bool})
 applabels = pd.read_csv(os.path.join(datadir,'app_labels.csv'))
df_external = pd.merge(df_population, df_add, on='Country/Region', how='left')
<merge>
gatrain['trainrow'] = np.arange(gatrain.shape[0])
 gatest['testrow'] = np.arange(gatest.shape[0] )
df_pop = pd.merge(df, df_external, on=['Country/Region', 'Province/State'], how='left')
 df_pop<define_variables>
brandencoder = LabelEncoder().fit(phone.phone_brand)
 phone['brand'] = brandencoder.transform(phone['phone_brand'])
 gatrain['brand'] = phone['brand']
gatest['brand'] = phone['brand']
Xtr_brand = csr_matrix(( np.ones(gatrain.shape[0]), 
(gatrain.trainrow, gatrain.brand)))
 Xte_brand = csr_matrix(( np.ones(gatest.shape[0]), 
(gatest.testrow, gatest.brand)))
 print('Brand features: train shape {}, test shape {}'.format(Xtr_brand.shape, Xte_brand.shape))
ext_cols = ['LaborForceTotal', 'LaborForcePerCapita ', 'DeathRate', 'AirTrafficPassengersTotal',
'AirTrafficPassengersPerCapita', 'HospitalBedDensity', 'Obesity', 'OldPeople',
'PhysiciansDensity']
ext_cols = ['LaborForcePerCapita ', 'DeathRate', 'AirTrafficPassengersPerCapita', 'HospitalBedDensity', 'Obesity', 'OldPeople',
'PhysiciansDensity']
pop_cols = ['DayNum', 'RealDayNum', 'Yearly change', 'Density', 'Land Area', 'Med.Age', 'Urban Pop', 'Population']
pop_cols = ['DayNum', 'Med.Age', 'Urban Pop', 'Density']
model_x_columns_without_dummies = pop_cols + ext_cols + previous_days_cols + special_cols
model_x_columns = model_x_columns_without_dummies + continent_columns
def rmsle(ytrue, ypred):
return np.sqrt(mean_squared_log_error(ytrue, ypred))
def mae(ytrue, ypred):
return mean_absolute_error(ytrue, ypred)
 def analyse(data_y_test, data_y_pred):
chart_data = pd.DataFrame({'x1': data_y_test.flatten() ,
'x2': data_y_pred.flatten() ,
'y': np.abs(data_y_test.flatten() -data_y_pred.flatten() ).flatten() })
 sns.scatterplot(x='x1', y='y', data=chart_data, color='black')
 sns.scatterplot(x='x2', y='y', data=chart_data, color='red')
 print('RMSLE: {}'.format(round(rmsle(data_y_test, data_y_pred), 6)))
 def analyse2(tr_y, tr_pred, data_y_test, data_y_pred):
chart_data0 = pd.DataFrame({
'x00': tr_y.flatten() ,
'x01': tr_pred.flatten() ,
'y0': np.abs(tr_y.flatten() -tr_pred.flatten() ).flatten() })
 chart_data1 = pd.DataFrame({
'x10': data_y_test.flatten() ,
'x11': data_y_pred.flatten() ,
'y1': np.abs(data_y_test.flatten() -data_y_pred.flatten() ).flatten() })
 fig, ax =plt.subplots(1,2)
 sns.scatterplot(x='x00', y='y0', data=chart_data0, color='blue', ax=ax[0])
 sns.scatterplot(x='x01', y='y0', data=chart_data0, color='yellow', ax=ax[0])
 sns.scatterplot(x='x10', y='y1', data=chart_data1, color='black', ax=ax[1])
 sns.scatterplot(x='x11', y='y1', data=chart_data1, color='red', ax=ax[1])
 print('RMSLE train: {}'.format(round(rmsle(tr_y, tr_pred), 6)))
 print('RMSLE test: {}'.format(round(rmsle(data_y_test, data_y_pred), 6)))
 def prepare_data(df, what_to_predict, test_size=0.3, dropna=False):
df_tmp = df.copy()
if dropna:
df_tmp.dropna(inplace=True)
 data_X = df_tmp[model_x_columns]
data_y = np.log1p(df_tmp[[what_to_predict]].values.flatten())
 return train_test_split(data_X, data_y, test_size=test_size, random_state=42)
 def predict_output(input_data, model):
df_final = input_data[model_x_columns].copy()
y_pred = model.predict(df_final)
 return y_pred
def expm1_relu(y):
tmp = np.expm1(y)
 tmp[tmp<0]=0
return np.around(tmp )<train_model>
brandencoder = LabelEncoder().fit(phone.phone_brand)
 phone['brand'] = brandencoder.transform(phone['phone_brand'])
 gatrain['brand'] = phone['brand']
gatest['brand'] = phone['brand']
Xtr_brand = csr_matrix(( np.ones(gatrain.shape[0]), 
(gatrain.trainrow, gatrain.brand)))
 Xte_brand = csr_matrix(( np.ones(gatest.shape[0]), 
(gatest.testrow, gatest.brand)))
 print('Brand features: train shape {}, test shape {}'.format(Xtr_brand.shape, Xte_brand.shape))
data_X_tr, data_X_test, data_y_tr, data_y_test = prepare_data(df_pop, 'Fatalities', test_size=0.25, dropna=True)
 model_f = XGBRegressor(learning_rate=0.1, n_estimators=50, max_depth=3, min_child_weight=0, gamma=0,
subsample=0.65, colsample_bytree=0.65, reg_alpha=0, reg_lambda=0,
objective='reg:squarederror', scale_pos_weight=1, seed=37)
 model_f.fit(data_X_tr, data_y_tr)
 tr_pred = predict_output(data_X_tr, model_f)
 data_y_pred = predict_output(data_X_test, model_f)
 analyse2(expm1_relu(data_y_tr), expm1_relu(tr_pred), expm1_relu(data_y_test), expm1_relu(data_y_pred))<split>
m = phone.phone_brand.str.cat(phone.device_model)
 modelencoder = LabelEncoder().fit(m)
 phone['model'] = modelencoder.transform(m)
 gatrain['model'] = phone['model']
gatest['model'] = phone['model']
Xtr_model = csr_matrix(( np.ones(gatrain.shape[0]), 
(gatrain.trainrow, gatrain.model)))
 Xte_model = csr_matrix(( np.ones(gatest.shape[0]), 
(gatest.testrow, gatest.model)))
 print('Model features: train shape {}, test shape {}'.format(Xtr_model.shape, Xte_model.shape))
model_cc = XGBRegressor(learning_rate=0.08, n_estimators=400, max_depth=6, min_child_weight=0, gamma=0,
subsample=0.7, colsample_bytree=0.78, reg_alpha=0, reg_lambda=28,
objective='reg:squarederror', scale_pos_weight=1, seed=37)
 data_X_tr, data_X_test, data_y_tr, data_y_test = prepare_data(df_pop, 'ConfirmedCases', test_size=0.3, dropna=True)
 hist = model_cc.fit(data_X_tr, data_y_tr)
 tr_pred = predict_output(data_X_tr, model_cc)
 data_y_pred = predict_output(data_X_test, model_cc)
 analyse2(expm1_relu(data_y_tr), expm1_relu(tr_pred), expm1_relu(data_y_test), expm1_relu(data_y_pred))<feature_engineering>
appencoder = LabelEncoder().fit(appevents.app_id)
 appevents['app'] = appencoder.transform(appevents.app_id)
 napps = len(appencoder.classes_)
 deviceapps =(appevents.merge(events[['device_id']], how='left',left_on='event_id',right_index=True)
.groupby(['device_id','app'])['app'].agg(['size'])
.merge(gatrain[['trainrow']], how='left', left_index=True, right_index=True)
.merge(gatest[['testrow']], how='left', left_index=True, right_index=True)
.reset_index())
 deviceapps.head()
df_test = pd.read_csv(data_base + 'test.csv')
 df_test.rename(columns={'Province_State': 'Province/State', 'Country_Region': 'Country/Region'}, inplace=True)
 df_test['Province/State'].fillna('entire country', inplace=True)
 df_test['DayNum'] =(df_test['Date'].astype('datetime64[ns]')- day_zero ).apply(lambda x: int(x.days))
for c1, c2 in zip(special_cols1, special_cols2):
zero_days = pd.DataFrame(df.groupby(['Country/Region', 'Province/State', c1] ).size().reset_index() [['Country/Region', 'Province/State', c1]])
 zero_days.drop_duplicates(subset=['Country/Region', 'Province/State'], keep='last', inplace=True)
 df_test = df_test.merge(zero_days, on=['Country/Region', 'Province/State'], how='left')
 df_test<merge>
d = deviceapps.dropna(subset=['trainrow'])
 Xtr_app = csr_matrix(( np.ones(d.shape[0]),(d.trainrow, d.app)) ,
shape=(gatrain.shape[0],napps))
d = deviceapps.dropna(subset=['testrow'])
 Xte_app = csr_matrix(( np.ones(d.shape[0]),(d.testrow, d.app)) ,
shape=(gatest.shape[0],napps))
print('Apps data: train shape {}, test shape {}'.format(Xtr_app.shape, Xte_app.shape))
for c1, c2 in zip(special_cols1, special_cols2):
real_day_num = df_test['DayNum'] - df_test[c1] + 1
df_test[c2] = real_day_num - real_day_num.where(real_day_num<0 ).fillna(0)
 df_test_pop = pd.merge(df_test, df_external, on=['Country/Region', 'Province/State'], how='left')
 df_test_pop.fillna(df_test_pop.mean() , inplace=True)
 df_test_pop<merge>
applabels = applabels.loc[applabels.app_id.isin(appevents.app_id.unique())]
applabels['app'] = appencoder.transform(applabels.app_id)
 labelencoder = LabelEncoder().fit(applabels.label_id)
 applabels['label'] = labelencoder.transform(applabels.label_id)
 nlabels = len(labelencoder.classes_ )
output_columns = ['ConfirmedCases', 'Fatalities']
tmp_output_columns = ['ConfirmedCases_y', 'Fatalities_y']
last_training_day = df['DayNum'].max()
first_test_day = df_test['DayNum'].min()
train_test_keys = ['Country/Region', 'Province/State', 'DayNum']
df_test_pop = pd.merge(df_test_pop, df[df['DayNum']>=first_test_day][train_test_keys+output_columns], on=train_test_keys, how='left')
 df_test_pop[output_columns] = df_test_pop[output_columns].fillna(0 ).copy()
df_test_pop<predict_on_test>
applabels = applabels.loc[applabels.app_id.isin(appevents.app_id.unique())]
applabels['app'] = appencoder.transform(applabels.app_id)
 labelencoder = LabelEncoder().fit(applabels.label_id)
 applabels['label'] = labelencoder.transform(applabels.label_id)
 nlabels = len(labelencoder.classes_ )
last_test_day = df_test['DayNum'].max()
for day in range(last_training_day+1, last_test_day+1):
print('predicting day {}({} to go)'.format(day, last_test_day-day))
up_to_current_day = df_test_pop.where(df_test['DayNum']<=day ).dropna(subset=['Country/Region'])
 previous_days_columns = add_extra_features_from_previous_days(up_to_current_day, TAIL)
 up_to_current_day['ConfirmedCases'] = expm1_relu(predict_output(up_to_current_day[model_x_columns], model_cc))
up_to_current_day['Fatalities'] = expm1_relu(predict_output(up_to_current_day[model_x_columns], model_f))
tmp_dataset = up_to_current_day[up_to_current_day['DayNum']==day][train_test_keys+output_columns]
df_test_pop = pd.merge(df_test_pop, tmp_dataset, on=train_test_keys, how='left', suffixes=('', '_y'))
df_test_pop[tmp_output_columns] = df_test_pop[tmp_output_columns].fillna(0 ).copy()
df_test_pop['ConfirmedCases'] += df_test_pop['ConfirmedCases_y']
df_test_pop['Fatalities'] += df_test_pop['Fatalities_y']
df_test_pop.drop(columns=tmp_output_columns, inplace=True)
<filter>
devicelabels =(deviceapps[['device_id','app']] 
.merge(applabels[['app','label']])
.groupby(['device_id','label'])['app'].agg(['size'])
.merge(gatrain[['trainrow']], how='left', left_index=True, right_index=True)
.merge(gatest[['testrow']], how='left', left_index=True, right_index=True)
.reset_index())
 devicelabels.head()
up_to_current_day[up_to_current_day.isnull().any(axis=1)]<save_to_csv>
d = devicelabels.dropna(subset=['trainrow'])
 Xtr_label = csr_matrix(( np.ones(d.shape[0]),(d.trainrow, d.label)) ,
shape=(gatrain.shape[0],nlabels))
d = devicelabels.dropna(subset=['testrow'])
 Xte_label = csr_matrix(( np.ones(d.shape[0]),(d.testrow, d.label)) ,
shape=(gatest.shape[0],nlabels))
print('Labels data: train shape {}, test shape {}'.format(Xtr_label.shape, Xte_label.shape))
submission_columns = ['ForecastId', 'ConfirmedCases', 'Fatalities']
df_test_pop[submission_columns].to_csv('submission.csv', index=False)
 df_test_pop<import_modules>
Xtrain = hstack(( Xtr_brand, Xtr_model, Xtr_app, Xtr_label), format='csr')
 Xtest = hstack(( Xte_brand, Xte_model, Xte_app, Xte_label), format='csr')
 print('All features: train shape {}, test shape {}'.format(Xtrain.shape, Xtest.shape))
plotly.offline.init_notebook_mode()
%matplotlib inline
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)) )<load_from_csv>
Xtrain = hstack(( Xtr_brand, Xtr_model, Xtr_app, Xtr_label), format='csr')
 Xtest = hstack(( Xte_brand, Xte_model, Xte_app, Xte_label), format='csr')
 print('All features: train shape {}, test shape {}'.format(Xtrain.shape, Xtest.shape))
pd.set_option('mode.chained_assignment', None)
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-2/test.csv")
 train = pd.read_csv(".. /input/covid19-global-forecasting-week-2/train.csv")
 train['Province_State'].fillna('', inplace=True)
 test['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'])
 train = train.sort_values(['Country_Region','Province_State','Date'])
 test = test.sort_values(['Country_Region','Province_State','Date'] )<feature_engineering>
targetencoder = LabelEncoder().fit(gatrain.group)
 y = targetencoder.transform(gatrain.group)
 nclasses = len(targetencoder.classes_ )
FirstDate = train.groupby('Country_Region' ).min() ['Date'].unique() [0]
train['Last Confirm'] = train['ConfirmedCases'].shift(1)
 while train[(train['Last Confirm'] > train['ConfirmedCases'])&(train['Date'] > FirstDate)].shape[0] > 0:
train['Last Confirm'] = train['ConfirmedCases'].shift(1)
 train['Last Fatalities'] = train['Fatalities'].shift(1)
 train.loc[(train['Last Confirm'] > train['ConfirmedCases'])&(train['Date'] > FirstDate),'ConfirmedCases'] = train.loc[(train['Last Confirm'] > train['ConfirmedCases'])&(train['Date'] > FirstDate),'Last Confirm']
train.loc[(train['Last Fatalities'] > train['Fatalities'])&(train['Date'] > FirstDate),'Fatalities'] = train.loc[(train['Last Fatalities'] > train['Fatalities'])&(train['Date'] > FirstDate),'Last Fatalities']
train['Last Confirm'] = train['ConfirmedCases'].shift(1)
 train['Last Fatalities'] = train['Fatalities'].shift(1 )<feature_engineering>
import xgboost as xgb
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
with tqdm(total=len(train['Country_Region'].unique())) as pbar:
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 feature_use = X_pred.columns[0]
for i in range(X_pred.shape[1] - 1,0,-1):
if(X_pred.iloc[0,i] > 0):
feature_use = X_pred.columns[i]
break
idx = X_train[X_train[feature_use] == 0].shape[0]
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
adjusted_X_pred = X_pred[feature_use].values.reshape(-1, 1)
 model = make_pipeline(PolynomialFeatures(2), BayesianRidge())
 model.fit(adjusted_X_train,adjusted_y_train_confirmed)
 y_hat_confirmed = model.predict(adjusted_X_pred)
 model.fit(adjusted_X_train,adjusted_y_train_fatalities)
 y_hat_fatalities = model.predict(adjusted_X_pred)
 pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 pbar.update(1)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_1 = df_val.copy()<compute_test_metric>
dtrain = xgb.DMatrix(Xtrain, y )
RMSLE(df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases'].values,df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases_hat'].values )<compute_test_metric>
params = {
"eta": 0.1,
"booster": "gblinear",
"objective": "multi:softprob",
"alpha": 4,
"lambda": 0,
"silent": 1,
"seed": 1233,
"num_class": 12,
"eval_metric": "mlogloss"
}
RMSLE(df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities'].values,df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities_hat'].values )<groupby>
xgb.cv(params, dtrain,
num_boost_round=50,
maximize = False )
country = "Vietnam"
df_val = df_val_1
df_val[df_val['Country_Region'] == country].groupby(['Date','Country_Region'] ).sum().reset_index()<save_model>
model = RandomForestClassifier()
animator.save('confirm_animation.gif', writer='imagemagick', fps=2)
 display(Image(url='confirm_animation.gif'))<feature_engineering>
model = xgb.train(params, dtrain, num_boost_round=25 )
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
with tqdm(total=len(train['Country_Region'].unique())) as pbar:
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
with warnings.catch_warnings() :
warnings.filterwarnings("ignore")
 df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 feature_use = X_pred.columns[0]
for i in range(X_pred.shape[1] - 1,0,-1):
if(X_pred.iloc[0,i] > 0):
feature_use = X_pred.columns[i]
break
idx = X_train[X_train[feature_use] == 0].shape[0]
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
max_train_date = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].max()
min_test_date = pred_data['Date'].min()
model = ExponentialSmoothing(adjusted_y_train_confirmed, trend = 'additive' ).fit()
y_hat_confirmed = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_confirmed = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['ConfirmedCases'].values
y_hat_confirmed = np.concatenate(( y_train_confirmed,y_hat_confirmed), axis = 0)
 model = ExponentialSmoothing(adjusted_y_train_fatalities, trend = 'additive' ).fit()
y_hat_fatalities = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_fatalities = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['Fatalities'].values
y_hat_fatalities = np.concatenate(( y_train_fatalities,y_hat_fatalities), axis = 0)
 pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 pbar.update(1)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_2 = df_val.copy()<feature_engineering>
dtest = xgb.DMatrix(Xtest )
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
with tqdm(total=len(train['Country_Region'].unique())) as pbar:
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
with warnings.catch_warnings() :
warnings.filterwarnings("ignore")
 df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 feature_use = X_pred.columns[0]
for i in range(X_pred.shape[1] - 1,0,-1):
if(X_pred.iloc[0,i] > 0):
feature_use = X_pred.columns[i]
break
idx = X_train[X_train[feature_use] == 0].shape[0]
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
idx = X_pred[X_pred[feature_use] == 0].shape[0]
adjusted_X_pred = X_pred[idx:][feature_use].values.reshape(-1, 1)
 pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
max_train_date = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].max()
min_test_date = pred_data['Date'].min()
model = SARIMAX(adjusted_y_train_confirmed, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_confirmed = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_confirmed = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['ConfirmedCases'].values
y_hat_confirmed = np.concatenate(( y_train_confirmed,y_hat_confirmed), axis = 0)
 model = SARIMAX(adjusted_y_train_fatalities, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_fatalities = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_fatalities = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['Fatalities'].values
y_hat_fatalities = np.concatenate(( y_train_fatalities,y_hat_fatalities), axis = 0)
 pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 pbar.update(1)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_3 = df_val.copy()<compute_test_metric>
model.predict(dtest)
 pred = pd.DataFrame(model.predict(dtest), index = gatest.index, columns=targetencoder.classes_ )
<save_to_csv><EOS> 
pred.to_csv('xgb_subm.csv',index=True )