Kaggle competition: COVID19 Global Forecasting (Week 2)
train['encoded_category'], test['encoded_category'] = target_encode(train['project_subject_categories'], test['project_subject_categories'], train['project_is_approved'])
 train['encoded_subcategory'], test['encoded_subcategory'] = target_encode(train['project_subject_subcategories'], test['project_subject_subcategories'], train['project_is_approved'] )<randomize_order>
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)) )
train = train.reindex(
 np.random.permutation(train.index))<define_variables>
pd.set_option('mode.chained_assignment', None)
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-2/test.csv")
 train = pd.read_csv(".. /input/covid19-global-forecasting-week-2/train.csv")
 train['Province_State'].fillna('', inplace=True)
 test['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'])
 train = train.sort_values(['Country_Region','Province_State','Date'])
 test = test.sort_values(['Country_Region','Province_State','Date'] )
training_examples = preprocess_features(train.head(122080))
training_targets = preprocess_targets(train.head(122080))
validation_examples = preprocess_features(train.tail(60000))
validation_targets = preprocess_targets(train.tail(60000))
test_examples = preprocess_features(test.copy() )<feature_engineering>
train[['ConfirmedCases', 'Fatalities']] = train.groupby(['Country_Region', 'Province_State'])[['ConfirmedCases', 'Fatalities']].transform('cummax' )
vectorizer=TfidfVectorizer(stop_words=stop)
 vectorizer.fit(train['project_subject_categories'])
 train_project_subject_categories = vectorizer.transform(training_examples['project_subject_categories'])
 validation_project_subject_categories = vectorizer.transform(validation_examples['project_subject_categories'])
 test_project_subject_categories = vectorizer.transform(test_examples['project_subject_categories'])
 vectorizer.fit(train['project_subject_subcategories'])
 train_project_subject_subcategories = vectorizer.transform(training_examples['project_subject_subcategories'])
 validation_project_subject_subcategories = vectorizer.transform(validation_examples['project_subject_subcategories'])
 test_project_subject_subcategories = vectorizer.transform(test_examples['project_subject_subcategories'] )<feature_engineering>
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
with tqdm(total=len(train['Country_Region'].unique())) as pbar:
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 feature_use = X_pred.columns[0]
for i in range(X_pred.shape[1] - 1,0,-1):
if(X_pred.iloc[0,i] > 0):
feature_use = X_pred.columns[i]
break
idx = X_train[X_train[feature_use] == 0].shape[0]
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
adjusted_X_pred = X_pred[feature_use].values.reshape(-1, 1)
 model = make_pipeline(PolynomialFeatures(2), BayesianRidge())
 model.fit(adjusted_X_train,adjusted_y_train_confirmed)
 y_hat_confirmed = model.predict(adjusted_X_pred)
 model.fit(adjusted_X_train,adjusted_y_train_fatalities)
 y_hat_fatalities = model.predict(adjusted_X_pred)
 pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 pbar.update(1)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_1 = df_val.copy()
vectorizer=TfidfVectorizer(stop_words=stop, ngram_range=(1, 2), max_df=0.9, min_df=5, max_features=2000)
 vectorizer.fit(train['project_title'])
 train_project_title = vectorizer.transform(training_examples['project_title'])
 validation_project_title = vectorizer.transform(validation_examples['project_title'])
 test_project_title = vectorizer.transform(test_examples['project_title'])
 vectorizer.fit(train['project_resource_summary'])
 train_project_resource_summary = vectorizer.transform(training_examples['project_resource_summary'])
 validation_project_resource_summary = vectorizer.transform(validation_examples['project_resource_summary'])
 test_project_resource_summary = vectorizer.transform(test_examples['project_resource_summary'] )<feature_engineering>
RMSLE(df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases'].values,df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases_hat'].values )
vectorizer=TfidfVectorizer(stop_words=stop, ngram_range=(1, 3), max_df=0.9, min_df=5, max_features=2000)
 vectorizer.fit(train['project_essay_1'])
 train_project_essay_1 = vectorizer.transform(training_examples['project_essay_1'])
 validation_project_essay_1 = vectorizer.transform(validation_examples['project_essay_1'])
 test_project_essay_1 = vectorizer.transform(test_examples['project_essay_1'])
 vectorizer.fit(train['project_essay_2'])
 train_project_essay_2 = vectorizer.transform(training_examples['project_essay_2'])
 validation_project_essay_2 = vectorizer.transform(validation_examples['project_essay_2'])
 test_project_essay_2 = vectorizer.transform(test_examples['project_essay_2'] )<feature_engineering>
RMSLE(df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities'].values,df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities_hat'].values )
def get_polarity(text):
textblob = TextBlob(text)
 pol = textblob.sentiment.polarity
return round(pol,3)
 def get_subjectivity(text):
textblob = TextBlob(text)
 subj = textblob.sentiment.subjectivity
return round(subj,3 )<feature_engineering>
method_list = ['Poly Bayesian Ridge','SARIMA']
method_val = [df_val_1,df_val_3]
for i in range(0,2):
df_val = method_val[i]
method_score = [method_list[i]] + [RMSLE(df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases'].values,df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases_hat'].values)] + [RMSLE(df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities'].values,df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities_hat'].values)]
print(method_score )
training_examples['polarity1'] = training_examples['project_essay_1'].apply(get_polarity)
 training_examples['subjectivity1'] = training_examples['project_essay_1'].apply(get_subjectivity)
 training_examples['polarity2'] = training_examples['project_essay_2'].apply(get_polarity)
 training_examples['subjectivity2'] = training_examples['project_essay_2'].apply(get_subjectivity )<feature_engineering>
df_val = df_val_1
submission = df_val[['ForecastId','ConfirmedCases_hat','Fatalities_hat']]
submission.columns = ['ForecastId','ConfirmedCases','Fatalities']
submission.to_csv('submission.csv', index=False)
 submission