Kaggle competition: COVID19 Global Forecasting (Week 2)
for row in meaneduc_nan.iterrows() :
idx=row[0]
idhogar=row[1]['idhogar']
m=me[me['idhogar']==idhogar]['escolari'].tolist() [0]
data.at[idx, 'meaneduc']=m
data.at[idx, 'SQBmeaned']=m*m 
<count_missing_values>
train_data = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/train.csv")
 test_data = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/test.csv" )
data['v2a1'].isnull().sum()<count_values>
for row in range(len(train_data['Province_State'])) :
if str(train_data['Province_State'][row])== 'nan':
train_data['Province_State'][row] = train_data['Country_Region'][row]
norent=data[data['v2a1'].isnull() ]
print("Owns his house:", norent[norent['tipovivi1']==1]['Id'].count())
 print("Owns his house paying installments", norent[norent['tipovivi2']==1]['Id'].count())
 print("Rented ", norent[norent['tipovivi3']==1]['Id'].count())
 print("Precarious ", norent[norent['tipovivi4']==1]['Id'].count())
 print("Other ", norent[norent['tipovivi5']==1]['Id'].count())
 print("Total ", 6860 )<feature_engineering>
for row in range(len(test_data['Province_State'])) :
if str(test_data['Province_State'][row])== 'nan':
test_data['Province_State'][row] = test_data['Country_Region'][row]
data['v2a1']=data['v2a1'].fillna(0 )<count_missing_values>
day_number = train_data[train_data['Province_State'] == 'Florida'].shape[0]
day_number
data['v18q1'].isna().sum()<count_values>
country_number = train_data.shape[0]/day_number
country_number
tabletnan=data[data['v18q1'].isnull() ]
tabletnan[tabletnan['v18q']==0]['Id'].count()<feature_engineering>
state_code = 0
train_data['Province_State'][0] = 0
for instance in range(0, int(country_number)) :
for state_index in range(0, day_number):
train_data['Province_State'][state_index +(day_number * instance)] = state_code
state_code += 1
data['v18q1']=data['v18q1'].fillna(0 )<count_missing_values>
train_data[train_data['Country_Region'] == 'Turkey']
data['rez_esc'].isnull().sum()<filter>
train_data[train_data['Country_Region'] == 'Turkey']['Date']
data[data['rez_esc']>1][['age', 'escolari', 'rez_esc']][:20]<filter>
time1 = train_data[train_data['Country_Region'] == 'Turkey']['Date'].values[-1]
time2 = '2020-04-30'
FMT = '%Y-%m-%d'
time_diff = datetime.strptime(time2, FMT)- datetime.strptime(time1, FMT )
rez_esc_nan=data[data['rez_esc'].isnull() ]
rez_esc_nan[(rez_esc_nan['age']<18)& rez_esc_nan['escolari']>0][['age', 'escolari']]<feature_engineering>
future_day_number = time_diff.days
future_day_number
data['rez_esc']=data['rez_esc'].fillna(0 )<define_variables>
train_data = train_data.drop('Country_Region', axis = 1)
 test_data = test_data.drop('Country_Region', axis = 1 )
d={}
weird=[]
for row in data.iterrows() :
idhogar=row[1]['idhogar']
target=row[1]['Target']
if idhogar in d:
if d[idhogar]!=target:
weird.append(idhogar)
 else:
d[idhogar]=target<define_variables>
CC_dataset = pd.DataFrame()
Fatal_dataset = pd.DataFrame()
target = []
for forecast_id in range(0, int(country_number)) :
sample_CC_data = train_data.loc[train_data['Province_State'] == forecast_id]
sample_Fatal_data = train_data.loc[train_data['Province_State'] == forecast_id]
CC_column = sample_CC_data['ConfirmedCases']
sample_CC_data = sample_CC_data.drop(['ConfirmedCases', 'Date', 'Fatalities'], axis = 1)
 Fatal_column = sample_Fatal_data['Fatalities']
sample_Fatal_data = sample_Fatal_data.drop(['Fatalities', 'Date', 'ConfirmedCases'], axis = 1)
 sample_CC_data = sample_CC_data.head(1)
 sample_Fatal_data = sample_Fatal_data.head(1)
 CC_list = list(CC_column)
 Fatal_list = list(Fatal_column)
 for CC_index in range(0,len(CC_list)) :
sample_CC_data['CC' + str(CC_index+1)] = CC_list[CC_index]
for Fatal_index in range(0,len(Fatal_list)) :
sample_Fatal_data['F' + str(Fatal_index+1)] = Fatal_list[Fatal_index]
CC_dataset = CC_dataset.append(sample_CC_data)
 Fatal_dataset = Fatal_dataset.append(sample_Fatal_data )
len(set(weird))<filter>
country_number = CC_dataset.shape[0]
country_number
data[data['idhogar']==weird[2]][['idhogar','parentesco1', 'Target']]<feature_engineering>
CC_daily_dataset = pd.DataFrame()
Fatality_daily_dataset = pd.DataFrame()
for state_id in range(0, country_number):
sample_case = CC_dataset.loc[CC_dataset['Province_State'] == state_id]
sample_case['Day1CC'] = sample_case['CC1']
sample_fatality = Fatal_dataset.loc[Fatal_dataset['Province_State'] == state_id]
sample_fatality['Day1F'] = sample_fatality['F1']
for case_index in range(1, day_number):
sample_case['Day' + str(case_index + 1)+ 'CC'] = sample_case['CC' + str(case_index + 1)] - sample_case['CC' + str(case_index)]
sample_case = sample_case.drop(['CC' + str(case_index)], axis = 1)
 sample_fatality['Day' + str(case_index + 1)+ 'F'] = sample_fatality['F' + str(case_index + 1)] - sample_fatality['F' + str(case_index)]
sample_fatality = sample_fatality.drop(['F' + str(case_index)], axis = 1)
 sample_case = sample_case.drop(['CC' + str(day_number)], axis = 1)
 sample_fatality = sample_fatality.drop(['F' + str(day_number)], axis = 1)
 CC_daily_dataset = CC_daily_dataset.append(sample_case)
 Fatality_daily_dataset = Fatality_daily_dataset.append(sample_fatality)
for i in set(weird):
hhold=data[data['idhogar']==i][['idhogar', 'parentesco1', 'Target']]
target=hhold[hhold['parentesco1']==1]['Target'].tolist() [0]
for row in hhold.iterrows() :
idx=row[0]
if row[1]['parentesco1']!=1:
data.at[idx, 'Target']=target<filter>
last_day_CC = CC_dataset.iloc[:,-1].values
last_day_F = Fatal_dataset.iloc[:,-1].values
data[data['idhogar']==weird[1]][['idhogar','parentesco1', 'Target']]<categorify>
CC_daily_dataset[CC_daily_dataset['Province_State'] == 223]
def data_cleaning(data):
data['dependency']=np.sqrt(data['SQBdependency'])
 data['rez_esc']=data['rez_esc'].fillna(0)
 data['v18q1']=data['v18q1'].fillna(0)
 data['v2a1']=data['v2a1'].fillna(0)
 conditions = [ 
(data['edjefe']=='no')&(data['edjefa']=='no'), 
(data['edjefe']=='yes')&(data['edjefa']=='no'), 
(data['edjefe']=='no')&(data['edjefa']=='yes'), 
(data['edjefe']!='no')&(data['edjefe']!='yes')&(data['edjefa']=='no'), 
(data['edjefe']=='no')&(data['edjefa']!='no')
 ]
choices = [0, 1, 1, data['edjefe'], data['edjefa']]
data['edjefx']=np.select(conditions, choices)
 data['edjefx']=data['edjefx'].astype(int)
 data.drop(['edjefe', 'edjefa'], axis=1, inplace=True)
 meaneduc_nan=data[data['meaneduc'].isnull() ][['Id','idhogar','escolari']]
me=meaneduc_nan.groupby('idhogar')['escolari'].mean().reset_index()
for row in meaneduc_nan.iterrows() :
idx=row[0]
idhogar=row[1]['idhogar']
m=me[me['idhogar']==idhogar]['escolari'].tolist() [0]
data.at[idx, 'meaneduc']=m
data.at[idx, 'SQBmeaned']=m*m
return data<set_options>
CC_daily_dataset = CC_daily_dataset.drop(['Id', 'Province_State'], axis = 1)
 Fatality_daily_dataset = Fatality_daily_dataset.drop(['Id', 'Province_State'], axis = 1 )
%matplotlib inline 
<drop_column>
features_CC = CC_daily_dataset.iloc[:, :-1*(future_day_number)]
target_CC = CC_daily_dataset.iloc[:, -1*(future_day_number):]
features_F = CC_daily_dataset.iloc[:, :-1*(future_day_number)]
target_F = Fatality_daily_dataset.iloc[:, -1*(future_day_number):]
train_features_CC, test_features_CC, train_target_CC, test_target_CC = train_test_split(features_CC, target_CC,
test_size = 0.2)
 train_features_F, test_features_F, train_target_F, test_target_F = train_test_split(features_F, target_F,
test_size = 0.2)
data_undersampled=data.drop(data.query('Target == 4' ).sample(frac=.75 ).index )<prepare_x_and_y>
train_features_CC = np.array(train_features_CC)
 train_features_F = np.array(train_features_F)
 test_features_CC = np.array(test_features_CC)
 test_features_F = np.array(test_features_F)
 train_target_CC = np.array(train_target_CC)
 train_target_F = np.array(train_target_F)
 test_target_CC = np.array(test_target_CC)
 test_target_F = np.array(test_target_F)
 features_CC = np.array(features_CC)
 features_F = np.array(features_F)
 target_CC = np.array(target_CC)
 target_F = np.array(target_F )
X=data_undersampled.drop(['Id', 'idhogar', 'Target', 'edjefe', 'edjefa'], axis=1)
 y=data_undersampled['Target']<split>
random_number = random.randint(0, 10000)
 model_CC = LinearRegression()
train_features_CC, test_features_CC, train_target_CC, test_target_CC = train_test_split(features_CC, target_CC,
test_size = 0.2,
random_state = random_number)
 regressor = model_CC.fit(train_features_CC, train_target_CC)
 prediction_CC = regressor.predict(features_CC )
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2 )<import_modules>
for country_index in range(country_number):
for day_index in range(len(prediction_CC[country_index])) :
prediction_CC[country_index][day_index] += target_CC[country_index][-1]
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV<choose_model_class>
random_number = random.randint(0, 10000)
 model_F = LinearRegression()
train_features_F, test_features_F, train_target_F, test_target_F = train_test_split(features_F, target_F,
test_size = 0.2,
random_state = random_number)
 regressor_F = model_F.fit(train_features_F, train_target_F)
 prediction_F = regressor_F.predict(features_F )
clf = RandomForestClassifier()
params={'n_estimators': list(range(40,61, 1)) }
gs = GridSearchCV(clf, params, cv=5 )<train_model>
for country_index in range(country_number):
for day_index in range(len(prediction_F[country_index])) :
prediction_F[country_index][day_index] += target_F[country_index][-1]
gs.fit(X_train, y_train )<predict_on_test>
for index in range(len(prediction_CC)) :
for inner_index in range(len(prediction_CC[index])) :
if prediction_CC[index][inner_index] >= 0:
prediction_CC[index][inner_index] = round(prediction_CC[index][inner_index])
 elif prediction_CC[index][inner_index] < 0:
prediction_CC[index][inner_index] = 0
preds=gs.predict(X_test )<compute_test_metric>
for index in range(len(prediction_F)) :
for inner_index in range(len(prediction_F[index])) :
if prediction_F[index][inner_index] >= 0:
prediction_F[index][inner_index] = round(prediction_F[index][inner_index])
 elif prediction_F[index][inner_index] < 0:
prediction_F[index][inner_index] = 0
print(classification_report(y_test, preds))<compute_test_metric>
for index in range(len(prediction_CC)) :
prediction_CC[index][0] = last_day_CC[index]
for inner_index in range(1, len(prediction_CC[index])) :
prediction_CC[index][inner_index] = prediction_CC[index][inner_index] + prediction_CC[index][inner_index-1]
prediction_CC
print(confusion_matrix(y_test, preds))<find_best_params>
for index in range(len(prediction_F)) :
prediction_F[index][0] = last_day_F[index]
for inner_index in range(1, len(prediction_F[index])) :
prediction_F[index][inner_index] = prediction_F[index][inner_index] + prediction_F[index][inner_index-1]
prediction_F
print(gs.best_params_)
 print(gs.best_score_)
 print(gs.best_estimator_ )<compute_test_metric>
target_CC = CC_dataset.iloc[:,-13:]
target_CC = np.array(target_CC)
 target_CC
cvres = gs.cv_results_
for mean_score, params in zip(cvres["mean_test_score"], cvres["params"]):
print(np.sqrt(mean_score), params )<load_from_csv>
target_F = Fatal_dataset.iloc[:,-13:]
target_F = np.array(target_F)
 target_F
test_data=pd.read_csv('.. /input/test.csv' )<drop_column>
confirmed_cases = np.concatenate(( target_CC, prediction_CC), axis = 1)
 fatalities = np.concatenate(( target_F, prediction_F), axis = 1 )
ids=test_data['Id']
test_data.drop(['Id', 'idhogar'], axis=1, inplace=True )<predict_on_test>
submitted_data = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/submission.csv" )
test_predictions=gs.predict(test_data )<create_dataframe>
dim_CC = confirmed_cases.shape[0]*confirmed_cases.shape[1]
dim_F = fatalities.shape[0]*fatalities.shape[1]
submit=pd.DataFrame({'Id': ids, 'Target': test_predictions} )<save_to_csv>
confirmed_cases = confirmed_cases.reshape(dim_CC,)
 fatalities = fatalities.reshape(dim_F, )
submit.to_csv('submit.csv', index=False )<set_options>
submitted_data['Fatalities'] = fatalities
submitted_data['ConfirmedCases'] = confirmed_cases
%matplotlib inline
plt.style.use('fivethirtyeight')
 plt.rcParams['font.size'] = 18
plt.rcParams['patch.edgecolor'] = 'k'<load_from_csv>
submitted_data.to_csv('submission.csv', index = False, encoding = 'utf-8-sig' )