Kaggle competition: COVID19 Global Forecasting (Week 4)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-4)<feature_engineering>
warnings.filterwarnings('ignore')
n=14
commits_df.loc[n,'commit_num'] = 18
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 248
commits_df.loc[n,'hidden_dim_third'] = 248
commits_df.loc[n,'LB_score'] = 0.25841<feature_engineering>
covid_cases = pd.read_csv('.. /input/novel-corona-virus-2019-dataset/covid_19_data.csv')
 covid_cases.head()
n=15
commits_df.loc[n,'commit_num'] = 19
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 248
commits_df.loc[n,'hidden_dim_third'] = 384
commits_df.loc[n,'LB_score'] = 0.25956<feature_engineering>
training_data = pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 testing_data = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv" )
n=16
commits_df.loc[n,'commit_num'] = 20
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 128
commits_df.loc[n,'hidden_dim_third'] = 248
commits_df.loc[n,'LB_score'] = 0.25891<feature_engineering>
print(training_data.isnull().sum())
 print(testing_data.isnull().sum())
 print(training_data.dtypes)
 print(testing_data.dtypes)
 training_data['Province_State'].fillna("",inplace = True)
 testing_data['Province_State'].fillna("",inplace = True )
n=17
commits_df.loc[n,'commit_num'] = 21
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 248
commits_df.loc[n,'hidden_dim_third'] = 240
commits_df.loc[n,'LB_score'] = 0.25831<feature_engineering>
country_list = covid_cases['Country/Region'].unique()
country_grouped_covid = covid_cases[0:1]
for country in country_list:
test_data = covid_cases['Country/Region'] == country
test_data = covid_cases[test_data]
country_grouped_covid = pd.concat([country_grouped_covid, test_data], axis=0)
 country_grouped_covid.reset_index(drop=True)
 country_grouped_covid.head()
country_grouped_covid.drop('Last Update', axis=1, inplace=True)
 country_grouped_covid['Province/State'].replace(np.nan, "Not Reported", inplace=True)
 country_grouped_covid.head()
n=18
commits_df.loc[n,'commit_num'] = 22
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 248
commits_df.loc[n,'hidden_dim_third'] = 216
commits_df.loc[n,'LB_score'] = 0.25827<feature_engineering>
latest_data = country_grouped_covid['ObservationDate'] == '04/10/2020'
country_data = country_grouped_covid[latest_data]
country_list = country_data['Country/Region'].unique()
print("The total number of countries with COVID-19 Confirmed cases = {}".format(country_list.size))
n=19
commits_df.loc[n,'commit_num'] = 23
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 248
commits_df.loc[n,'hidden_dim_third'] = 184
commits_df.loc[n,'LB_score'] = 0.25898<feature_engineering>
folder_name = '.. /input/covcsd-covid19-countries-statistical-dataset'
file_type = 'csv'
seperator =','
dataframe = pd.concat([pd.read_csv(f, sep=seperator)for f in glob.glob(folder_name + "/*."+file_type)],ignore_index=True,sort=False )
n=20
commits_df.loc[n,'commit_num'] = 24
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 248
commits_df.loc[n,'hidden_dim_third'] = 208
commits_df.loc[n,'LB_score'] = 0.25868<feature_engineering>
covid_data = dataframe[['Date', 'State', 'Country', 'Cumulative_cases', 'Cumulative_death',
'Daily_cases', 'Daily_death', 'Latitude', 'Longitude', 'Temperature',
'Min_temperature', 'Max_temperature', 'Wind_speed', 'Precipitation',
'Fog_Presence', 'Population', 'Population Density/km', 'Median_Age',
'Sex_Ratio', 'Age%_65+', 'Hospital Beds/1000', 'Available Beds/1000',
'Confirmed Cases/1000', 'Lung Patients(F)', 'Lung Patients(M)',
'Life Expectancy(M)', 'Life Expectancy(F)', 'Total_tests_conducted',
'Out_Travels(mill.) ', 'In_travels(mill.) ', 'Domestic_Travels(mill.) ']]
n=21
commits_df.loc[n,'commit_num'] = 25
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 248
commits_df.loc[n,'hidden_dim_third'] = 212
commits_df.loc[n,'LB_score'] = 0.25823<feature_engineering>
training_data['Country_Region'] = training_data['Country_Region'] + ' ' + training_data['Province_State']
testing_data['Country_Region'] = testing_data['Country_Region'] + ' ' + testing_data['Province_State']
del training_data['Province_State']
del testing_data['Province_State']
def split_date(date):
date = date.split('-')
 date[0] = int(date[0])
 if(date[1][0] == '0'):
date[1] = int(date[1][1])
 else:
date[1] = int(date[1])
 if(date[2][0] == '0'):
date[2] = int(date[2][1])
 else:
date[2] = int(date[2])
 return date
training_data.Date = training_data.Date.apply(split_date)
 testing_data.Date = testing_data.Date.apply(split_date )
n=22
commits_df.loc[n,'commit_num'] = 26
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 248
commits_df.loc[n,'hidden_dim_third'] = 210
commits_df.loc[n,'LB_score'] = 0.25844<feature_engineering>
year = []
month = []
day = []
for i in training_data.Date:
year.append(i[0])
 month.append(i[1])
 day.append(i[2])
 training_data['Year'] = year
training_data['Month'] = month
training_data['Day'] = day
del training_data['Date']
year = []
month = []
day = []
for i in testing_data.Date:
year.append(i[0])
 month.append(i[1])
 day.append(i[2])
 testing_data['Year'] = year
testing_data['Month'] = month
testing_data['Day'] = day
del testing_data['Date']
del training_data['Id']
del testing_data['ForecastId']
del testing_data['Year']
del training_data['Year']
n=23
commits_df.loc[n,'commit_num'] = 27
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 64
commits_df.loc[n,'hidden_dim_second'] = 124
commits_df.loc[n,'hidden_dim_third'] = 108
commits_df.loc[n,'LB_score'] = 0.25963<feature_engineering>
latest_data = covid_data['Date'] == '30-03-2020'
country_data_detailed = covid_data[latest_data]
country_data_detailed.drop(['Daily_cases','Daily_death','Latitude','Longitude'],axis=1,inplace=True)
 country_data_detailed.head(3 )
n=24
commits_df.loc[n,'commit_num'] = 28
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 248
commits_df.loc[n,'hidden_dim_third'] = 108
commits_df.loc[n,'LB_score'] = 0.25860<feature_engineering>
country_data_detailed['Lung Patients(F)'].replace('Not reported',np.nan,inplace=True)
 country_data_detailed['Lung Patients(F)'] = country_data_detailed['Lung Patients(F)'].astype("float" )
n=25
commits_df.loc[n,'commit_num'] = 29
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 244
commits_df.loc[n,'hidden_dim_third'] = 128
commits_df.loc[n,'LB_score'] = 0.25846<feature_engineering>
temperature_data = pd.read_csv('.. /input/covcsd-covid19-countries-statistical-dataset/temperature_data.csv')
 temperature_data.head()
commits_df['LB_score'] = pd.to_numeric(commits_df['LB_score'])
 commits_df['best'] = 0
commits_df.loc[commits_df['LB_score'].idxmin() , 'best'] = 1<sort_values>
sample = temperature_dataset['Temperature'].sample(n=250)
 test = temperature_dataset['Temperature']
stat, p = ttest_ind(sample, test)
 print('Statistics=%.3f, p=%.3f' %(stat, p))
commits_df.sort_values(by=['LB_score'] )<load_from_csv>
training_data['ConfirmedCases'] = training_data['ConfirmedCases'].apply(int)
 training_data['Fatalities'] = training_data['Fatalities'].apply(int)
 cases = training_data.ConfirmedCases
fatalities = training_data.Fatalities
del training_data['ConfirmedCases']
del training_data['Fatalities']
lb = LabelEncoder()
training_data['Country_Region'] = lb.fit_transform(training_data['Country_Region'])
 testing_data['Country_Region'] = lb.transform(testing_data['Country_Region'])
 scaler = MinMaxScaler()
x_train = scaler.fit_transform(training_data.values)
 x_test = scaler.transform(testing_data.values )
train = pd.read_json('/kaggle/input/stanford-covid-vaccine/train.json', lines=True)
 test = pd.read_json('/kaggle/input/stanford-covid-vaccine/test.json', lines=True)
 sample_sub = pd.read_csv("/kaggle/input/stanford-covid-vaccine/sample_submission.csv" )<define_variables>
rf = XGBRegressor(n_estimators = 1500 , max_depth = 15, learning_rate=0.1)
 rf.fit(x_train,cases)
 cases_pred = rf.predict(x_test)
 rf = XGBRegressor(n_estimators = 1500 , max_depth = 15, learning_rate=0.1)
 rf.fit(x_train,fatalities)
 fatalities_pred = rf.predict(x_test )
target_cols = ['reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']<categorify>
cases_pred = np.around(cases_pred)
 fatalities_pred = np.around(fatalities_pred)
 cases_pred[cases_pred < 0] = 0
fatalities_pred[fatalities_pred < 0] = 0
token2int = {x:i for i, x in enumerate('().ACGUBEHIMSX')}
def get_pair_index_structure(structure):
structure = np.array([struc for struc in structure], dtype="<U4")
 open_index = np.where(structure == "(")[0]
closed_index = np.where(structure == ")")[0]
structure[open_index] = range(0, len(open_index))
structure[closed_index] = range(len(open_index)-1, -1, -1)
 structure[structure == "."] = -1
structure = structure.astype(int)
 pair_structure = np.array([-1]*len(structure))
for i in range(len(open_index)) :
start, end = np.where(structure == i)[0]
pair_structure[start] = end
pair_structure[end] = start
return pair_structure<categorify>
submission_dataset = pd.read_csv(".. /input/covid19-global-forecasting-week-4/submission.csv")
 submission_dataset['ConfirmedCases'] = cases_pred
submission_dataset['Fatalities'] = fatalities_pred
submission_dataset.head()
def preprocess_inputs(df, cols=['sequence', 'structure', 'predicted_loop_type']):
return np.transpose(
 np.array(
 df[cols] 
.applymap(lambda seq: [token2int[x] for x in seq])
.values 
.tolist() 
), 
(0, 2, 1)
)
 train_inputs_all = preprocess_inputs(train)
 train_labels_all = np.array(train[target_cols].values.tolist() ).transpose(( 0, 2, 1))<compute_test_metric>
submission_dataset.to_csv("submission.csv" , index = False )