Kaggle competition: COVID19 Global Forecasting (Week 2)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-2)<groupby>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit
from statsmodels.tsa.statespace.sarimax import SARIMAX
from statsmodels.tsa.arima_model import ARIMA
from random import random
households_leader = train.groupby('idhogar')['parentesco1'].sum()
households_no_head = train.loc[train['idhogar'].isin(households_leader[households_leader == 0].index), :]
print('There are {} households without a head.'.format(households_no_head['idhogar'].nunique()))<groupby>
df=pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/train.csv",
usecols=['Province_State','Country_Region','Date','ConfirmedCases','Fatalities'])
households_no_head_equal = households_no_head.groupby('idhogar')['Target'].apply(lambda x: x.nunique() == 1)
 print('{} Households with no head have different labels.'.format(sum(households_no_head_equal == False)) )<feature_engineering>
df.fillna(' ',inplace=True)
 df['Lat']=df['Province_State']+df['Country_Region']
df.drop('Province_State',axis=1,inplace=True)
 df.drop('Country_Region',axis=1,inplace=True)
 df.head()
for household in not_equal.index:
true_target = int(train[(train['idhogar'] == household)&(train['parentesco1'] == 1.0)]['Target'])
 train.loc[train['idhogar'] == household, 'Target'] = true_target
all_equal = train.groupby('idhogar')['Target'].apply(lambda x: x.nunique() == 1)
 not_equal = all_equal[all_equal != True]
print('There are {} households where the family members do not all have the same target.'.format(len(not_equal)) )<feature_engineering>
countries_list=df.Lat.unique()
df1=[]
for i in countries_list:
df1.append(df[df['Lat']==i])
 print("we have "+ str(len(df1)) +" regions in our dataset")
 test=pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/test.csv" )
missing = pd.DataFrame(data.isnull().sum() ).rename(columns = {0: 'total'})
 missing['percent'] = missing['total'] / len(data)
 missing.sort_values('percent', ascending = False ).head(10 ).drop('Target' )<groupby>
submit_confirmed=[]
submit_fatal=[]
for i in df1:
data = i.ConfirmedCases.astype('int32' ).tolist()
try:
model = SARIMAX(data, order=(1,1,0), seasonal_order=(1,1,0,12),measurement_error=True)
 model_fit = model.fit(disp=False)
 predicted = model_fit.predict(len(data), len(data)+34)
 new=np.concatenate(( np.array(data),np.array([int(num)for num in predicted])) ,axis=0)
 submit_confirmed.extend(list(new[-43:]))
except:
submit_confirmed.extend(list(data[-10:-1]))
for j in range(34):
submit_confirmed.append(data[-1]*2)
 data = i.Fatalities.astype('int32' ).tolist()
try:
model = SARIMAX(data, order=(1,1,0), seasonal_order=(1,1,0,12),measurement_error=True)
 model_fit = model.fit(disp=False)
 predicted = model_fit.predict(len(data), len(data)+34)
 new=np.concatenate(( np.array(data),np.array([int(num)for num in predicted])) ,axis=0)
 submit_fatal.extend(list(new[-43:]))
except:
submit_fatal.extend(list(data[-10:-1]))
for j in range(34):
submit_fatal.append(data[-1]*2 )
heads.groupby('v18q')['v18q1'].apply(lambda x: x.isnull().sum() )<data_type_conversions>
df_submit=pd.concat([pd.Series(np.arange(1,1+len(submit_confirmed))),pd.Series(submit_confirmed),pd.Series(submit_fatal)],axis=1)
 df_submit=df_submit.fillna(method='pad' ).astype(int)
 df_submit.head()
data['v18q1'] = data['v18q1'].fillna(0 )<feature_engineering>
df_submit.rename(columns={0: 'ForecastId', 1: 'ConfirmedCases',2: 'Fatalities',}, inplace=True)
 complete_test= pd.merge(test, df_submit, how="left", on="ForecastId")
 complete_test.head()
data.loc[(data['tipovivi1'] == 1), 'v2a1'] = 0
data['v2a1-missing'] = data['v2a1'].isnull()
data['v2a1-missing'].value_counts()<feature_engineering>
df_submit.to_csv('submission.csv',header=['ForecastId','ConfirmedCases','Fatalities'],index=False)
 complete_test.to_csv('complete_test.csv',index=False )