Kaggle competition: West Nile Virus Prediction
<SOS> metric: AUC Kaggle data source: west-nile-virus-prediction<load_from_csv>
import pandas as pd
import numpy as np
import math
import scipy.stats as sps
from time import time
from sklearn import preprocessing, ensemble, metrics, feature_selection, model_selection, pipeline
import xgboost as xgb
from IPython.display import display
from matplotlib import pyplot
dftrain = pd.read_csv('.. /input/covid19-global-forecasting-week-2/train.csv', parse_dates=['Date'] ).sort_values(by=['Country_Region', 'Date'] ).fillna('None')
 dftest = pd.read_csv('.. /input/covid19-global-forecasting-week-2/test.csv', parse_dates=['Date'] ).sort_values(by=['Country_Region', 'Date'] ).fillna('None')
 dfsubm = pd.read_csv('.. /input/covid19-global-forecasting-week-2/submission.csv' )<load_from_csv>
dateparse = lambda x: pd.datetime.strptime(x, '%Y-%m-%d')
 dtype_map_weather = dict(Station = 'str')
 dtype_map_test_train = dict(Block = 'str', Street = 'str')
 test = pd.read_csv('.. /input/test.csv', parse_dates=['Date'], date_parser=dateparse, dtype= dtype_map_test_train)
 train = pd.read_csv('.. /input/train.csv', parse_dates=['Date'], date_parser=dateparse, dtype= dtype_map_test_train)
 weather = pd.read_csv('.. /input/weather.csv', parse_dates=['Date'], date_parser=dateparse, dtype= dtype_map_weather)
 sample_sub = pd.read_csv('.. /input/sampleSubmission.csv' )
confirmed = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_confirmed_global.csv' ).sort_values(by='Country/Region')
 deaths = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_deaths_global.csv')
 recovered = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_recovered_global.csv' )<feature_engineering>
weather_exclude = ['Dewpoint', 'WetBulb', 'CodeSum', 'Depth', 'Water1', 'SnowFall', 'StnPressure',
'SeaLevel', 'ResultSpeed', 'ResultDir', 'AvgSpeed','DewPoint']
weather_cols = [col for col in weather.columns if col not in weather_exclude]
weather = weather[weather_cols]
train_exclude = ['Address', 'AddressNumberAndStreet', 'AddressAccuracy', 'NumMosquitos']
train_cols = [col for col in train.columns if col not in train_exclude]
train = train[train_cols]
test_exclude = ['Address', 'AddressNumberAndStreet', 'AddressAccuracy', 'Id']
test_cols = [col for col in test.columns if col not in test_exclude]
test = test[test_cols]
confirmed['Country_Region'] = confirmed['Country/Region']
confirmed['Province_State'] = confirmed['Province/State']
confirmed.head()<merge>
miss_weather = ['M', '-']
trace_weather = ['T']
dftrain = dftrain.join(confirmed[['Country_Region', 'Province_State', 'Lat', 'Long']].set_index(['Province_State', 'Country_Region']), on=['Province_State', 'Country_Region'] )<feature_engineering>
cols_not_date = [col for col in weather.columns if col != 'Date']
dftrain['Dayofyear'] = dftrain['Date'].dt.dayofyear
dftest['Dayofyear'] = dftest['Date'].dt.dayofyear<data_type_conversions>
weather[cols_not_date].apply(pd.value_counts, axis=1)[miss_weather + trace_weather].fillna(0 ).sum()
def transpose_df(df):
df = df.drop(['Lat','Long'],axis=1 ).groupby('Country/Region' ).sum().T
df.index = pd.to_datetime(df.index)
 return df<feature_engineering>
check = weather[cols_not_date].apply(pd.value_counts, axis=0 ).fillna(0)
 check.loc[['M', '-', 'T']]
confirmedT = transpose_df(confirmed)
 deathsT = transpose_df(deaths)
 recoveredT = transpose_df(recovered)
 mortalityT = deathsT/confirmedT<feature_engineering>
check_stat1 = weather[cols_not_date][weather.Station == '1'].apply(pd.value_counts, axis=0 ).fillna(0)
 check_stat1.loc[['M', '-', 'T']]
def add_day(df):
df['Date'] = df.index
df['Dayofyear'] = df['Date'].dt.dayofyear
return df<drop_column>
check_stat2 = weather[cols_not_date][weather.Station == '2'].apply(pd.value_counts, axis=0 ).fillna(0)
 check_stat2.loc[['M', '-', 'T']]
confirmedT, deathsT, recoveredT, mortalityT = add_day(confirmedT), add_day(deathsT), add_day(recoveredT), add_day(mortalityT )<sort_values>
check.loc[['M', '-', 'T']]/(len(weather)) * 100
allcountries_ordered = confirmed.set_index(['Country_Region'] ).iloc[:,-2].sort_values(ascending=False ).index.tolist()<save_to_csv>
check_stat1.loc[['M', '-', 'T']]/(len(weather)) * 100
confirmed.set_index(['Country_Region'] ).iloc[:,-2].sort_values(ascending=False ).to_csv('confirmed_countries.csv' )<feature_engineering>
check_stat2.loc[['M', '-', 'T']]/(len(weather)) * 100
def df_day1(df, confirmed):
df_day1 = pd.DataFrame({'Days since 100 cases' : np.arange(1000)} ).set_index('Days since 100 cases')
 countries_df = df.columns.tolist() [:-2]
countries_conf = confirmed.columns.tolist() [:-2]
for ic, country in enumerate(countries_df):
for ic2, country2 in enumerate(countries_conf):
if country == country2:
dfsub = df[confirmed[country] > 100.][country]
df_day1[country] = np.nan
df_day1.loc[:len(dfsub)-1,country] =(dfsub ).tolist()
df_day1 = df_day1.dropna(how='all')
 return df_day1<define_variables>
weather = weather.replace('M', np.NaN)
 weather = weather.replace('-', np.NaN)
 weather = weather.replace('T', 0.005)
 weather.Tmax = weather.Tmax.fillna(method = 'ffill')
 weather.Tmin = weather.Tmin.fillna(method = 'ffill')
 weather.Depart = weather.Depart.fillna(method = 'ffill')
 weather.Heat = weather.Heat.fillna(method = 'ffill')
 weather.Cool = weather.Cool.fillna(method = 'ffill')
 weather.PrecipTotal = weather.PrecipTotal.fillna(method = 'ffill' )
confirmed_day1 = df_day1(confirmedT, confirmedT)
 deaths_day1 = df_day1(deathsT, confirmedT)
 recovered_day1 = df_day1(recoveredT, confirmedT)
 mortality_day1 = df_day1(mortalityT, confirmedT)
 confirmednorm_day1 = confirmed_day1/confirmed_day1.loc[0,:]
maxday = confirmed_day1.shape[0]<data_type_conversions>
to_numeric = ['Tmax','Tmin','Tavg', 'Depart', 'Heat', 'Cool', 'PrecipTotal']
for col in to_numeric:
weather[col]= pd.to_numeric(weather[col] )
date_day1 = confirmedT.copy()
for column in date_day1:
date_day1[column] = confirmedT.index.tolist()
date_day1 = df_day1(date_day1, confirmedT )<compute_train_metric>
weather.Sunrise = weather.Sunrise.fillna(method = 'ffill')
 weather.Sunset = weather.Sunset.fillna(method = 'ffill' )
def logistic_curve(x, k, x_0, ymax):
return ymax /(1 + np.exp(-k*(x-x_0)) )<compute_train_metric>
counter = 0
tracker = []
for index, val in enumerate(weather.Sunset):
try:
pd.to_datetime(val, format = '%H%M' ).time()
except:
counter += 1
tracker.append(( index, val, val[2:], counter))
print(tracker[-1])
def logistic_curve2(x, k1, k2, x_0, ymax):
return ymax /(1 + np.exp(-k1*(x-x_0)) + np.exp(-k2*(x-x_0)) )<data_type_conversions>
time_func = lambda x: pd.Timestamp(pd.to_datetime(x, format = '%H%M'))
list_countries = dftrain[dftrain['Date'] == '2020-01-22']['Country_Region'].tolist()
list_states = dftrain[dftrain['Date'] == '2020-01-22']['Province_State'].tolist()
datenow = datetime.now()<data_type_conversions>
weather.Sunrise = weather.Sunrise.apply(time_func )
list_date_pand = [] ; list_maxcases = []; list_maxfat = []
for country, state in list(zip(list_countries, list_states)) :
df2 = dftrain.loc[(dftrain['Country_Region'] == country)&(dftrain['Province_State'] == state)].fillna('None')
 maxcases, maxfat = df2['ConfirmedCases'].max() , df2['Fatalities'].max()
date_pand2 = []
date_pand = df2[df2['ConfirmedCases'] > 100.]['Date'].tolist()
try:
list_date_pand.append(pd.to_datetime(date_pand[0]))
except:
list_date_pand.append(pd.to_datetime(datenow))
list_maxcases.append(maxcases); list_maxfat.append(maxfat)
 dfstartpand = pd.DataFrame(np.array([list_countries, list_states, list_date_pand, list_maxcases, list_maxfat] ).T,
columns=['Country_Region', 'Province_State', 'Start_Pandemic', 'ConfirmedCases', 'Fatalities'])
 dfstartpand['Start_Pandemic'] = dfstartpand['Start_Pandemic'].dt.date<sort_values>
weather.Sunset = weather.Sunset.apply(time_func )
dfstartpand_ordered = dfstartpand.sort_values(by=['Start_Pandemic', 'ConfirmedCases', 'Fatalities'], ascending=[True, False, False])
 country_state_ordered = list(zip(dfstartpand_ordered['Country_Region'].tolist() , dfstartpand_ordered['Province_State']))
datetrain = dftrain['Date'].unique()
datetest = dftest['Date'].unique()<define_variables>
minutes=(weather.Sunset - weather.Sunrise ).astype('timedelta64[m]' )
dftest['ConfirmedCases_logreg'] = 0.0 ; dftrain['ConfirmedCases_logreg'] = 0.0
dftest['Fatalities_logreg'] = 0.0 ; dftrain['Fatalities_logreg'] = 0.0
p0 = 1
for country, state in country_state_ordered:
masktrain =(dftrain['Country_Region'] == country)&(dftrain['Province_State'] == state)
 masktrain2 =(dftrain['Country_Region'] == country)&(dftrain['Province_State'] == state)&(dftrain['Date'] <= '2020-03-31')&(dftrain['Date'] >= starttest)
 masktest =(dftest['Country_Region'] == country)&(dftest['Province_State'] == state)
 masktest2 =(dftest['Country_Region'] == country)&(dftest['Province_State'] == state)&(dftest['Date'] <= '2020-03-31')
 df2plot = dftrain[masktrain].set_index('Date')
 X = np.arange(len(df2plot))
X_test =(np.timedelta64(datetest[0]-datetrain[0], 'D')).astype(float)+np.arange(0,len(datetest))
try:
y = df2plot['ConfirmedCases']
p0_cases = [1/(len(X)/2.) , X[-1], y.max() ]
popt, pcov = curve_fit(logistic_curve, X, y, p0=p0_cases,bounds=([0,0,0],np.inf), maxfev=1000)
 k_cases, x_0_cases, ymax_cases = popt
cases_train_fc = pd.Series(logistic_curve(X, k_cases, x_0_cases, ymax_cases),index=df2plot.index)
 cases_test_fc = pd.Series(logistic_curve(X_test, k_cases, x_0_cases, ymax_cases),index=datetest)
 dftest.loc[masktest,'ConfirmedCases_logreg'] = cases_test_fc.tolist()
dftrain.loc[masktrain,'ConfirmedCases_logreg'] = cases_train_fc.tolist()
except:
print(country+' '+state+' Unable to fit the confirmed cases')
 dftest.loc[masktest,'ConfirmedCases_logreg'] = dftrain.loc[masktrain,'ConfirmedCases'].iloc[-1]
dftrain.loc[masktrain,'ConfirmedCases_logreg'] = dftrain.loc[masktrain,'ConfirmedCases']
try:
y = df2plot['Fatalities']
p0_deaths = [1/(len(X)/2.) , X[-1], y.max() ]
popt, pcov = curve_fit(logistic_curve, X, y, p0=p0_deaths,bounds=([0,0,0],np.inf), maxfev=1000)
 k_deaths, x_0_deaths, ymax_deaths = popt
deaths_train_fc = pd.Series(logistic_curve(X, k_deaths, x_0_deaths, ymax_deaths),index=datetrain)
 deaths_test_fc = pd.Series(logistic_curve(X_test, k_deaths, x_0_deaths, ymax_deaths),index=datetest)
 dftest.loc[masktest,'Fatalities_logreg'] = deaths_test_fc.tolist()
dftrain.loc[masktrain,'Fatalities_logreg'] = deaths_train_fc.tolist()
except:
print(country+' '+state+' Unable to fit the fatalities')
 dftest.loc[masktest,'Fatalities_logreg'] = dftrain.loc[masktrain,'Fatalities'].iloc[-1]
dftrain.loc[masktrain,'Fatalities_logreg'] = dftrain.loc[masktrain,'Fatalities']
dftest.loc[masktest2,'ConfirmedCases_logreg'] = dftrain.loc[masktrain2,'ConfirmedCases'].tolist()
dftest.loc[masktest2,'Fatalities_logreg'] = dftrain.loc[masktrain2,'Fatalities'].tolist()<data_type_conversions>
hours = minutes/60
dfsubm['ConfirmedCases'] = dftest['ConfirmedCases_logreg']
dfsubm['Fatalities'] = dftest['Fatalities_logreg']<save_to_csv>
weather['DayLength_MPrec'] =(weather.Sunset - weather.Sunrise ).astype('timedelta64[m]')/60
dfsubm.to_csv('submission.csv', index=False )<define_variables>
weather['DayLength_NearH'] = np.round(((weather.Sunset - weather.Sunrise ).astype('timedelta64[m]')/60 ).values )
DATASET_DIR = ".. /input/covid19-global-forecasting-week-2"
TRAIN_FILE = DATASET_DIR + "/train.csv"
TEST_FILE = DATASET_DIR + "/test.csv"<set_options>
weather['NightLength_MPrec']= 24.0 - weather.DayLength_MPrec
%matplotlib inline 
<load_from_csv>
weather['NightLength_NearH']= 24.0 - weather.DayLength_NearH
df_train = pd.read_csv(TRAIN_FILE)
 df_train.head()<load_from_csv>
hours_RiseSet_func = lambda x: x.minute/60.0 + float(x.hour )
df_test = pd.read_csv(TEST_FILE)
 df_test.head(5 )<count_missing_values>
weather['Sunrise_hours'] = weather.Sunrise.apply(hours_RiseSet_func )
df_train["Province_State"].isnull().sum()<count_missing_values>
weather['Sunset_hours'] = weather.Sunset.apply(hours_RiseSet_func )
df_train.isnull().sum()<filter>
mean_func = lambda x: x.mean()
blend_cols = ['Tmax', 'Tmin', 'Depart' ,'Heat', 'Cool', 'PrecipTotal']
df_train[df_train["Province_State"].notnull() ]<data_type_conversions>
blended_cols= ['blended_' + col for col in blend_cols]
df_train['Date'] = pd.to_datetime(df_train['Date'], infer_datetime_format=True)
 df_test['Date'] = pd.to_datetime(df_test['Date'], infer_datetime_format=True )<data_type_conversions>
station_1 = weather[blend_cols][weather.Station == '1']
station_2 = weather[blend_cols][weather.Station == '2']
pd.plotting.register_matplotlib_converters()
grouped_data = df_train.groupby('Date')['Date', 'ConfirmedCases', 'Fatalities'].sum().reset_index()
grouped_data = grouped_data.sort_values(by=['Date'], ascending=True)
 grouped_data['ConfirmedCases'] = grouped_data['ConfirmedCases'].astype(int)
 grouped_data['Fatalities'] = grouped_data['Fatalities'].astype(int)
 grouped_data.head()<data_type_conversions>
station_blend = pd.DataFrame(( station_1.values + station_2.values)/2, columns= blended_cols )
df_train['Date'] = pd.to_datetime(df_train['Date'])
 grouped_data = df_train.groupby(['Date'],as_index=True ).agg({'ConfirmedCases': 'max','Fatalities': 'max'})
 grouped_data['ConfirmedCases'] = grouped_data['ConfirmedCases'].astype(int)
 grouped_data['Fatalities'] = grouped_data['Fatalities'].astype(int)
 display(grouped_data.head() )<data_type_conversions>
extract_2 = weather[weather.Station == '2'].reset_index(drop = True)
 extract_2.head()
grouped_data = df_train.groupby('Country_Region')['ConfirmedCases', 'Fatalities'].sum().reset_index()
grouped_data = grouped_data.sort_values(by=['ConfirmedCases'], ascending=False)
 grouped_data['ConfirmedCases'] = grouped_data['ConfirmedCases'].astype(int)
 grouped_data['Fatalities'] = grouped_data['Fatalities'].astype(int)
 grouped_data.head(10 )<sort_values>
extract_1 = weather[weather.Station == '1'].reset_index(drop = True)
 extract_1.head()
grouped_data = df_train.groupby('Country_Region')['ConfirmedCases', 'Fatalities'].sum().reset_index()
grouped_data_sort_confirmed_cases = grouped_data.sort_values(by=['ConfirmedCases', 'Fatalities'], ascending=False)[:10]
grouped_data_sort_fatalities = grouped_data.sort_values(by=['Fatalities', 'ConfirmedCases'], ascending=False)[:10]<data_type_conversions>
joined_1 = extract_1.join(station_blend)
 joined_2 = extract_2.join(station_blend )
train_data = df_train.copy()
test_data = df_test.copy()
train_data['Date'] = train_data['Date'].dt.strftime("%m%d" ).astype(int)
 test_data['Date'] = test_data['Date'].dt.strftime("%m%d" ).astype(int)
<data_type_conversions>
weather_blend = pd.concat([joined_1, joined_2] )
train_data['Province_State'] = train_data['Province_State'].fillna("N/D")
 test_data['Province_State'] = test_data['Province_State'].fillna("N/D" )<categorify>
month_func = lambda x: x.month
day_func= lambda x: x.day
day_of_year_func = lambda x: x.dayofyear
week_of_year_func = lambda x: x.week
train['month'] = train.Date.apply(month_func)
 train['day'] = train.Date.apply(day_func)
 train['day_of_year'] = train.Date.apply(day_of_year_func)
 train['week'] = train.Date.apply(week_of_year_func)
 test['month'] = test.Date.apply(month_func)
 test['day'] = test.Date.apply(day_func)
 test['day_of_year'] = test.Date.apply(day_of_year_func)
 test['week'] = test.Date.apply(week_of_year_func )
LE=LabelEncoder()
train_data['Province_State']=LE.fit_transform(train_data['Province_State'])
 test_data['Province_State']=LE.transform(test_data['Province_State'])
 train_data['Country_Region']=LE.fit_transform(train_data['Country_Region'])
 test_data['Country_Region']=LE.transform(test_data['Country_Region'] )<prepare_x_and_y>
weather_blend = weather_blend.drop(['Sunrise', 'Sunset'], axis= 1 )
x_cols = ['Date','Province_State', 'Country_Region']
y_cols = ['ConfirmedCases', 'Fatalities']
train_data[x_cols].head(10 )<train_model>
train = train.merge(weather_blend, on='Date')
 test = test.merge(weather_blend, on='Date' )
model_dtr = DecisionTreeRegressor(max_depth=None)
 param_grid = {
'n_estimators': [100, 200, 300],
'learning_rate': [0.1, 0.01, 0.001]
}
model_abr = AdaBoostRegressor(base_estimator=model_dtr)
 model = RandomizedSearchCV(estimator = model_abr, param_distributions = param_grid,
n_iter = 100, cv =10, verbose=0, random_state=2020, n_jobs = -1)
 model.fit(train_data[x_cols], train_data[y_cols[0]])
 predictions1 = model.predict(test_data[x_cols])
 model = RandomizedSearchCV(estimator = model_abr, param_distributions = param_grid,
n_iter = 100, cv = 10, verbose=0, random_state=2020, n_jobs = -1)
 model.fit(train_data[x_cols], train_data[y_cols[1]])
 predictions2 = model.predict(test_data[x_cols] )<create_dataframe>
cols_to_write = [col for col in train.columns if col != 'Date']
submission = pd.DataFrame({'ForecastId':test_data['ForecastId'],'ConfirmedCases':predictions1,'Fatalities':predictions2})
 submission['ConfirmedCases'] = submission['ConfirmedCases'].astype(int)
 submission['Fatalities'] = submission['Fatalities'].astype(int)
 submission.head(50 )<save_to_csv>
train_station_1= train[train.Station == '1']
train_station_2= train[train.Station == '2']
test_station_1= test[test.Station == '1']
test_station_2= test[test.Station == '2']
filename = 'submission.csv'
submission.to_csv(filename,index=False)
 print('Saved file: ' + filename )<compute_test_metric>
train.to_csv('train.csv' )
%matplotlib inline
def sigmoid_sqrt_func(x, a, b, c, d, e):
return c + d /(1.0 + np.exp(-a*x+b)) + e*x**0.5
def sigmoid_linear_func(x, a, b, c, d, e):
return c + d /(1.0 + np.exp(-a*x+b)) + e*0.1*x
def sigmoid_quad_func(x, a, b, c, d, e, f):
return c + d /(1.0 + np.exp(-a*x+b)) + e*0.1*x + f*0.001*x*x
def sigmoid_func(x, a, b, c, d):
return c + d /(1.0 + np.exp(-a*x+b))
def exp_func(x, a, b, c, d):
return c + d * np.exp(a*x+b)
 def func_fitting(y, func=sigmoid_func, x_scale=50.0, y_scale=10000.0, start_pred=8, AN=0, MAXN=60, PN=15, b=5):
x = range(len(y))
x_real = np.array(x)/x_scale
y_real = np.array(y)/y_scale
x_train = x_real
y_train = y_real
def next_day_pred(AN, BN):
x_train = x_real[AN:BN]
y_train = y_real[AN:BN]
popt, pcov = curve_fit(func, x_train, y_train,
method='trf',
maxfev=20000,
p0=(1, 0, 0, 1),
bounds=[(-b, -np.inf, -np.inf, -b),(b, np.inf, np.inf, b)], 
)
 x_pred = np.array(range(MAXN)) /x_scale
y_pred = func(x_pred, *popt)
 return x_pred, y_pred
NP = start_pred
y_pred = [np.nan]*NP
y_pred_list = []
for BN in range(NP, len(y_real)) :
x_pred, y_pred_ = next_day_pred(BN-PN, BN)
 y_pred.append(y_pred_[BN])
 y_pred_list.append(y_pred_)
 for BN in range(len(y_real), len(y_pred_)) :
y_pred.append(y_pred_[BN])
 y_pred = np.array(y_pred)
 y_pred_list = np.array(y_pred_list)
 y_pred_std = np.std(y_pred_list[-2:], axis=0)
 return x_real*x_scale, y_real*y_scale, x_train*x_scale, y_train*y_scale, \
x_pred*x_scale, y_pred*y_scale, y_pred_std*y_scale
def draw_figure(start_date, title, x_real, y_real, x_train, y_train, x_pred, y_pred, y_pred_std):
def to_date(idx):
idx = np.round(idx)
 return datetime.datetime.strptime(start_date, '%m/%d/%Y' ).date() + datetime.timedelta(days=idx)
 fig, ax1 = plt.subplots(figsize=[14, 7])
 plot1 = ax1.plot(list(map(to_date, x_real)) , y_real, 'gs',label='original')
 plot2 = ax1.plot(list(map(to_date, x_pred)) , y_pred, 'r',label='predict')
 plot3 = ax1.fill_between(list(map(to_date, x_pred)) ,
np.maximum(0,(y_pred-y_pred_std)) , 
(y_pred+y_pred_std),
alpha=0.2, edgecolor='
plot0 = ax1.plot(list(map(to_date, x_train)) , y_train, 'y.',label='history')
 ax2=ax1.twinx()
ax2.plot(list(map(to_date, x_real)) [1:],(y_real[1:]-y_real[:-1]), '-s',label='original add')
 ax2.plot(list(map(to_date, x_pred)) [1:],(y_pred[1:]-y_pred[:-1]), '-',label='pred add')
 plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%m-%d'))
plt.gca().xaxis.set_major_locator(mdates.DayLocator())
 plt.gcf().autofmt_xdate()
plt.xlabel('x')
 plt.ylabel('y')
 fig.legend(loc=2)
 plt.title(title)
 plt.savefig('{}.pdf'.format(title))
plt.show()
date = list(map(to_date, x_pred))
pred = y_pred
real = y_real
for i in range(len(pred)) :
if i < len(real):
print('{}\t{:.0f}\t{:.0f}\t{:.3f}'.format(date[i], real[i], pred[i], np.abs(pred[i]-real[i])/real[i]*100))
else:
print('{}\t-\t{:.0f}'.format(date[i], pred[i]))
return pred 
<load_from_csv>
keep_cols = ['Date', u'Tmax', u'Tmin', u'Tavg',u'PrecipTotal']
train_station_2 = train_station_2[keep_cols]
test_station_2 = test_station_2[keep_cols]
prefix_s2 = 'stat_2_'
rename_cols_s2 = [prefix_s2 + col for col in train_station_2.columns]
train_station_2.columns = rename_cols_s2
test_station_2.columns = rename_cols_s2
train_data = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/train.csv')
 test_data = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/test.csv')
 pred_data = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/submission.csv')
 train_data = train_data.fillna(value='NULL')
 test_data = test_data.fillna(value='NULL' )<count_unique_values>
drop_cols = ['Heat', 'Cool', 'Depart', 'NightLength_MPrec', 'NightLength_NearH',
'blended_Depart', 'blended_Heat', 'blended_Cool']
train_station_1 = train_station_1.drop(drop_cols, axis= 1)
 test_station_1 = test_station_1.drop(drop_cols, axis= 1 )
train_date_list = train_data.iloc[:, 3].unique()
print(len(train_date_list))
print(train_date_list)
 test_date_list = test_data.iloc[:, 3].unique()
print(len(test_date_list))
print(test_date_list)
 len(train_data.groupby(['Province_State', 'Country_Region']))
len(test_data.groupby(['Province_State', 'Country_Region']))<load_from_csv>
prefix_s1 = 'stat_1_'
rename_cols_s1 = [prefix_s1 + col for col in keep_cols]
cols_to_rename= [col for col in train_station_1.columns if col in keep_cols]
s1_name_map = dict(zip(cols_to_rename, rename_cols_s1))
train_station_1 = train_station_1.rename(columns= s1_name_map)
 test_station_1 = test_station_1.rename(columns= s1_name_map )
start_date = '01/22/2020'
start_pred = 67
start_submit = 57
test_date_list = test_data.iloc[:, 3].unique()
test_data_filled = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/test.csv')
 test_data_filled = test_data_filled.fillna(value='NULL')
 test_data_filled['ConfirmedCases'] = pred_data['ConfirmedCases']
test_data_filled['Fatalities'] = pred_data['Fatalities']
for idx,(k, v)in enumerate(train_data.groupby(['Province_State', 'Country_Region'])) :
print(idx, k, end=' ')
 b_cc, b_f = 3, 2
if k[1] == 'China':
b_cc = 1
b_f = 1
if k[0] == 'Hong Kong':
b_cc = 3
elif k[1] == 'Italy':
b_cc = 5
b_f = 3
elif k[1] == 'US':
b_cc = 4
b_f = 2
elif k[1] == 'Spain':
b_cc = 4
b_f = 3
hist_num = v.loc[:,'ConfirmedCases'].tolist()
ret = func_fitting(hist_num, y_scale=max(1000, np.max(hist_num)) , b=b_cc,
start_pred=start_pred, PN=10, MAXN=len(hist_num)+30)
 ret = list(ret)
 real_cc = np.round(np.array(ret[1]))
pred_cc = np.round(np.array(ret[5]))
for i in range(len(real_cc)) :
pred_cc[i] = real_cc[i]
pred_cc = pred_cc[start_submit:]
print(pred_cc)
 hist_num = v.loc[:,'Fatalities'].tolist()
ret = func_fitting(hist_num, y_scale=max(1000, np.max(hist_num)) , b=b_f,
start_pred=start_pred, PN=10, MAXN=len(hist_num)+30)
 ret = list(ret)
 real_f = np.round(np.array(ret[1]))
pred_f = np.round(np.array(ret[5]))
for i in range(len(real_f)) :
pred_f[i] = real_f[i]
pred_f = pred_f[start_submit:]
print(pred_f)
 for i in range(14, len(pred_cc)) :
if pred_f[i] < 20 and pred_cc[i] > 200 and k[1] != 'China':
pred_f[i] = pred_cc[i] * 0.01 * np.log10(pred_cc[i])
 print(pred_f)
 print(pred_cc[-1], pred_f[-1])
 for i in range(len(pred_cc)) :
index =(test_data_filled['Province_State'] == k[0])& \ 
(test_data_filled['Country_Region'] == k[1])& \ 
(test_data_filled['Date'] == test_date_list[i])
 test_data_filled.loc[index, 'ConfirmedCases'] = pred_cc[i]
test_data_filled.loc[index, 'Fatalities'] = pred_f[i] 
<filter>
train_station_1 = train_station_1.reset_index(drop= True)
 train_station_2 = train_station_2.reset_index(drop = True)
 train_merge = pd.concat([train_station_1, train_station_2], axis= 1)
 test_station_1 = test_station_1.reset_index(drop= True)
 test_station_2 = test_station_2.reset_index(drop = True)
 test_merge = pd.concat([test_station_1, test_station_2], axis= 1 )
submission = test_data_filled.loc[:,['ForecastId', 'ConfirmedCases', 'Fatalities']]<save_to_csv>
labels = train_merge.pop('WnvPresent' ).values
submission.to_csv("submission.csv", index=False)
 submission.head(500 )<set_options>
train_merge = train_merge.drop(['stat_1_Date', 'stat_2_Date'], axis = 1)
 test_merge = test_merge.drop(['stat_1_Date', 'stat_2_Date' ], axis = 1 )
%matplotlib inline
InteractiveShell.ast_node_interactivity = "all"
pd.set_option('display.max_columns', 99)
 pd.set_option('display.max_rows', 99)
<set_options>
train_merge['Lat_int'] = train_merge.Latitude.astype(int)
 train_merge['Long_int'] = train_merge.Longitude.astype(int)
 test_merge['Lat_int'] = test_merge.Latitude.astype(int)
 test_merge['Long_int'] = test_merge.Longitude.astype(int )
plt.rcParams['figure.figsize'] = [16, 10]
plt.rcParams['font.size'] = 14
sns.set_palette(sns.color_palette('tab20', 20)) 
<load_from_csv>
train_merge = pd.get_dummies(train_merge, columns= ['Species'])
 train_merge = pd.get_dummies(train_merge, columns= ['Block'])
 train_merge = pd.get_dummies(train_merge, columns= ['Street'])
 train_merge = pd.get_dummies(train_merge, columns= ['Trap'])
 test_merge = pd.get_dummies(test_merge, columns= ['Species'])
 test_merge = pd.get_dummies(test_merge, columns= ['Block'])
 test_merge = pd.get_dummies(test_merge, columns= ['Street'])
 test_merge = pd.get_dummies(test_merge, columns= ['Trap'] )
COMP = '.. /input/covid19-global-forecasting-week-2'
DATEFORMAT = '%Y-%m-%d'
def get_comp_data(COMP):
train = pd.read_csv(f'{COMP}/train.csv')
 test = pd.read_csv(f'{COMP}/test.csv')
 submission = pd.read_csv(f'{COMP}/submission.csv')
 print(train.shape, test.shape, submission.shape)
 train['Country_Region'] = train['Country_Region'].str.replace(',', '')
 test['Country_Region'] = test['Country_Region'].str.replace(',', '')
 train['Location'] = train['Country_Region'] + '-' + train['Province_State'].fillna('')
 test['Location'] = test['Country_Region'] + '-' + test['Province_State'].fillna('')
 train['LogConfirmed'] = to_log(train.ConfirmedCases)
 train['LogFatalities'] = to_log(train.Fatalities)
 train = train.drop(columns=['Province_State'])
 test = test.drop(columns=['Province_State'])
 country_codes = pd.read_csv('.. /input/covid19-metadata/country_codes.csv', keep_default_na=False)
 train = train.merge(country_codes, on='Country_Region', how='left')
 test = test.merge(country_codes, on='Country_Region', how='left')
 train['DateTime'] = pd.to_datetime(train['Date'])
 test['DateTime'] = pd.to_datetime(test['Date'])
 return train, test, submission
def process_each_location(df):
dfs = []
for loc, df in tqdm(df.groupby('Location')) :
df = df.sort_values(by='Date')
 df['Fatalities'] = df['Fatalities'].cummax()
df['ConfirmedCases'] = df['ConfirmedCases'].cummax()
df['LogFatalities'] = df['LogFatalities'].cummax()
df['LogConfirmed'] = df['LogConfirmed'].cummax()
df['LogConfirmedNextDay'] = df['LogConfirmed'].shift(-1)
 df['ConfirmedNextDay'] = df['ConfirmedCases'].shift(-1)
 df['DateNextDay'] = df['Date'].shift(-1)
 df['LogFatalitiesNextDay'] = df['LogFatalities'].shift(-1)
 df['FatalitiesNextDay'] = df['Fatalities'].shift(-1)
 df['LogConfirmedDelta'] = df['LogConfirmedNextDay'] - df['LogConfirmed']
df['ConfirmedDelta'] = df['ConfirmedNextDay'] - df['ConfirmedCases']
df['LogFatalitiesDelta'] = df['LogFatalitiesNextDay'] - df['LogFatalities']
df['FatalitiesDelta'] = df['FatalitiesNextDay'] - df['Fatalities']
dfs.append(df)
 return pd.concat(dfs)
 def add_days(d, k):
return dt.datetime.strptime(d, DATEFORMAT)+ dt.timedelta(days=k)
 def to_log(x):
return np.log(x + 1)
 def to_exp(x):
return np.exp(x)- 1 
<save_to_csv>
train_merge= train_merge.drop('Station', axis= 1)
 test_merge= test_merge.drop('Station', axis= 1 )
train = train.sort_values(by='Date')
 countries_latest_state = train[train['Date'] == TRAIN_END].groupby([
'Country_Region', 'continent', 'geo_region', 'country_iso_code_3'] ).sum() [[
'ConfirmedCases', 'Fatalities']].reset_index()
countries_latest_state['Log10Confirmed'] = np.log10(countries_latest_state.ConfirmedCases + 1)
 countries_latest_state['Log10Fatalities'] = np.log10(countries_latest_state.Fatalities + 1)
 countries_latest_state = countries_latest_state.sort_values(by='Fatalities', ascending=False)
 countries_latest_state.to_csv('countries_latest_state.csv', index=False)
 countries_latest_state.shape
countries_latest_state.head()<merge>
latest_loc = train[train['Date'] == TRAIN_END][['Location', 'ConfirmedCases', 'Fatalities']]
max_loc = train.groupby(['Location'])[['ConfirmedCases', 'Fatalities']].max().reset_index()
check = pd.merge(latest_loc, max_loc, on='Location')
 np.mean(check.ConfirmedCases_x == check.ConfirmedCases_y)
 np.mean(check.Fatalities_x == check.Fatalities_y)
 check[check.Fatalities_x != check.Fatalities_y]
check[check.ConfirmedCases_x != check.ConfirmedCases_y]<feature_engineering>
unique_test_cols = [col for col in test_merge.columns if col not in train_merge.columns]
regional_progress = train_clean.groupby(['DateTime', 'continent'] ).sum() [['ConfirmedCases', 'Fatalities']].reset_index()
regional_progress['Log10Confirmed'] = np.log10(regional_progress.ConfirmedCases + 1)
 regional_progress['Log10Fatalities'] = np.log10(regional_progress.Fatalities + 1)
 regional_progress = regional_progress[regional_progress.continent != '<merge>
test_merge= test_merge.drop(unique_test_cols, axis= 1 )
countries_0301 = country_progress[country_progress.Date == '2020-03-01'][[
'Country_Region', 'ConfirmedCases', 'Fatalities']]
countries_0331 = country_progress[country_progress.Date == '2020-03-31'][[
'Country_Region', 'ConfirmedCases', 'Fatalities']]
countries_in_march = pd.merge(countries_0301, countries_0331, on='Country_Region', suffixes=['_0301', '_0331'])
 countries_in_march['IncreaseInMarch'] = countries_in_march.ConfirmedCases_0331 /(countries_in_march.ConfirmedCases_0301 + 1)
 countries_in_march = countries_in_march[countries_in_march.ConfirmedCases_0331 > 200].sort_values(
 by='IncreaseInMarch', ascending=False)
 countries_in_march.tail(15 )<save_to_csv>
train_clean['Geo
latest = train_clean[train_clean.Date == '2020-03-31'][[
'Geo
daily_confirmed_deltas = train_clean[train_clean.Date >= '2020-03-17'].pivot(
 'Geo
daily_confirmed_deltas = latest.merge(daily_confirmed_deltas, on='Geo
daily_confirmed_deltas.shape
daily_confirmed_deltas.head()
daily_confirmed_deltas.to_csv('daily_confirmed_deltas.csv', index=False )<save_to_csv>
clf = ensemble.RandomForestClassifier(n_estimators=1000, min_samples_split= 2, random_state= 42)
 clf.fit(train_merge, labels )
deltas = train_clean[np.logical_and(
 train_clean.LogConfirmed > 2,
~train_clean.Location.str.startswith('China')
)].dropna().sort_values(by='LogConfirmedDelta', ascending=False)
 deltas['start'] = deltas['LogConfirmed'].round(0)
 confirmed_deltas = pd.concat([
deltas.groupby('start')[['LogConfirmedDelta']].mean() ,
deltas.groupby('start')[['LogConfirmedDelta']].std() ,
deltas.groupby('start')[['LogConfirmedDelta']].count()
], axis=1)
 deltas.mean()
confirmed_deltas.columns = ['avg', 'std', 'cnt']
confirmed_deltas
confirmed_deltas.to_csv('confirmed_deltas.csv' )<feature_engineering>
predictions_randfor = clf.predict_proba(test_merge)[:,1]
DECAY = 0.93
DECAY ** 7, DECAY ** 14, DECAY ** 21, DECAY ** 28
confirmed_deltas = train.groupby(['Location', 'Country_Region', 'continent'])[[
'Id']].count().reset_index()
GLOBAL_DELTA = 0.11
confirmed_deltas['DELTA'] = GLOBAL_DELTA
confirmed_deltas.loc[confirmed_deltas.continent=='Africa', 'DELTA'] = 0.14
confirmed_deltas.loc[confirmed_deltas.continent=='Oceania', 'DELTA'] = 0.06
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Korea South', 'DELTA'] = 0.011
confirmed_deltas.loc[confirmed_deltas.Country_Region=='US', 'DELTA'] = 0.15
confirmed_deltas.loc[confirmed_deltas.Country_Region=='China', 'DELTA'] = 0.01
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Japan', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Singapore', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Taiwan*', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Switzerland', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Norway', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Iceland', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Austria', 'DELTA'] = 0.06
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Italy', 'DELTA'] = 0.04
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Spain', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Portugal', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Israel', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Iran', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Germany', 'DELTA'] = 0.07
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Malaysia', 'DELTA'] = 0.06
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Russia', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Ukraine', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Brazil', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Turkey', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Philippines', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Location=='France-', 'DELTA'] = 0.1
confirmed_deltas.loc[confirmed_deltas.Location=='United Kingdom-', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Location=='Diamond Princess-', 'DELTA'] = 0.00
confirmed_deltas.loc[confirmed_deltas.Location=='China-Hong Kong', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.Location=='San Marino-', 'DELTA'] = 0.03
confirmed_deltas.shape, confirmed_deltas.DELTA.mean()
confirmed_deltas[confirmed_deltas.DELTA != GLOBAL_DELTA].shape, confirmed_deltas[confirmed_deltas.DELTA != GLOBAL_DELTA].DELTA.mean()
confirmed_deltas[confirmed_deltas.DELTA != GLOBAL_DELTA]
confirmed_deltas.describe()<save_to_csv>
xgbc = xgb.XGBClassifier(seed= 42)
 xgbc.fit(train_merge, labels)
 fig, ax = pyplot.subplots(figsize=(10, 15))
xgb.plot_importance(xgbc, ax=ax)
daily_log_confirmed = train_clean.pivot('Location', 'Date', 'LogConfirmed' ).reset_index()
daily_log_confirmed = daily_log_confirmed.sort_values(TRAIN_END, ascending=False)
 daily_log_confirmed.to_csv('daily_log_confirmed.csv', index=False)
 for i, d in tqdm(enumerate(pd.date_range(add_days(TRAIN_END, 1), add_days(TEST_END, 1)))) :
new_day = str(d ).split(' ')[0]
last_day = dt.datetime.strptime(new_day, DATEFORMAT)- dt.timedelta(days=1)
 last_day = last_day.strftime(DATEFORMAT)
 for loc in confirmed_deltas.Location.values:
confirmed_delta = confirmed_deltas.loc[confirmed_deltas.Location == loc, 'DELTA'].values[0]
daily_log_confirmed.loc[daily_log_confirmed.Location == loc, new_day] = daily_log_confirmed.loc[daily_log_confirmed.Location == loc, last_day] + \
confirmed_delta * DECAY ** i<save_to_csv>
xgbc.get_fscore()
train_clean['Geo
latest = train_clean[train_clean.Date == TRAIN_END][[
'Geo
daily_death_deltas = train_clean[train_clean.Date >= '2020-03-17'].pivot(
 'Geo
daily_death_deltas = latest.merge(daily_death_deltas, on='Geo
daily_death_deltas.shape
daily_death_deltas.head()
daily_death_deltas.to_csv('daily_death_deltas.csv', index=False )<feature_engineering>
def calc_roc_auc(y, predict_probs):

 fpr, tpr, thresholds = metrics.roc_curve(y, predict_probs)
 roc_auc = metrics.auc(fpr, tpr)
 return roc_auc
death_deltas = train.groupby(['Location', 'Country_Region', 'continent'])[[
'Id']].count().reset_index()
GLOBAL_DELTA = 0.11
death_deltas['DELTA'] = GLOBAL_DELTA
death_deltas.loc[death_deltas.Country_Region=='China', 'DELTA'] = 0.005
death_deltas.loc[death_deltas.continent=='Oceania', 'DELTA'] = 0.08
death_deltas.loc[death_deltas.Country_Region=='Korea South', 'DELTA'] = 0.04
death_deltas.loc[death_deltas.Country_Region=='Japan', 'DELTA'] = 0.04
death_deltas.loc[death_deltas.Country_Region=='Singapore', 'DELTA'] = 0.05
death_deltas.loc[death_deltas.Country_Region=='Taiwan*', 'DELTA'] = 0.06
death_deltas.loc[death_deltas.Country_Region=='US', 'DELTA'] = 0.17
death_deltas.loc[death_deltas.Country_Region=='Switzerland', 'DELTA'] = 0.15
death_deltas.loc[death_deltas.Country_Region=='Norway', 'DELTA'] = 0.15
death_deltas.loc[death_deltas.Country_Region=='Iceland', 'DELTA'] = 0.01
death_deltas.loc[death_deltas.Country_Region=='Austria', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Country_Region=='Italy', 'DELTA'] = 0.07
death_deltas.loc[death_deltas.Country_Region=='Spain', 'DELTA'] = 0.1
death_deltas.loc[death_deltas.Country_Region=='Portugal', 'DELTA'] = 0.13
death_deltas.loc[death_deltas.Country_Region=='Israel', 'DELTA'] = 0.16
death_deltas.loc[death_deltas.Country_Region=='Iran', 'DELTA'] = 0.06
death_deltas.loc[death_deltas.Country_Region=='Germany', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Country_Region=='Malaysia', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Country_Region=='Russia', 'DELTA'] = 0.2
death_deltas.loc[death_deltas.Country_Region=='Ukraine', 'DELTA'] = 0.2
death_deltas.loc[death_deltas.Country_Region=='Brazil', 'DELTA'] = 0.2
death_deltas.loc[death_deltas.Country_Region=='Turkey', 'DELTA'] = 0.22
death_deltas.loc[death_deltas.Country_Region=='Philippines', 'DELTA'] = 0.12
death_deltas.loc[death_deltas.Location=='France-', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Location=='United Kingdom-', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Location=='Diamond Princess-', 'DELTA'] = 0.00
death_deltas.loc[death_deltas.Location=='China-Hong Kong', 'DELTA'] = 0.01
death_deltas.loc[death_deltas.Location=='San Marino-', 'DELTA'] = 0.05
death_deltas.shape
death_deltas.DELTA.mean()
death_deltas[death_deltas.DELTA != GLOBAL_DELTA].shape
death_deltas[death_deltas.DELTA != GLOBAL_DELTA].DELTA.mean()
death_deltas[death_deltas.DELTA != GLOBAL_DELTA]
death_deltas.describe()<save_to_csv>
train_split, val_split, label_train_split, label_val_split = model_selection.train_test_split(train_merge,
labels, test_size = 0.33, random_state = 42, stratify= labels )
daily_log_deaths = train_clean.pivot('Location', 'Date', 'LogFatalities' ).reset_index()
daily_log_deaths = daily_log_deaths.sort_values(TRAIN_END, ascending=False)
 daily_log_deaths.to_csv('daily_log_deaths.csv', index=False)
 for i, d in tqdm(enumerate(pd.date_range(add_days(TRAIN_END, 1), add_days(TEST_END, 1)))) :
new_day = str(d ).split(' ')[0]
last_day = dt.datetime.strptime(new_day, DATEFORMAT)- dt.timedelta(days=1)
 last_day = last_day.strftime(DATEFORMAT)
 for loc in death_deltas.Location:
death_delta = death_deltas.loc[death_deltas.Location == loc, 'DELTA'].values[0]
daily_log_deaths.loc[daily_log_deaths.Location == loc, new_day] = daily_log_deaths.loc[daily_log_deaths.Location == loc, last_day] + \
death_delta * DECAY ** i<feature_engineering>
def select_features_by_importance_threshold(model, X_train, y_train, selection_model, X_test, y_test,
minimum = False):
if minimum:
thresholds= np.unique(model.feature_importances_[model.feature_importances_ > minimum])
 thresholds = np.insert(thresholds, 0, 0.)
else:
thresholds= np.unique(model.feature_importances_)
 print(thresholds)
 for thresh in thresholds:
	
selection = feature_selection.SelectFromModel(model, threshold=thresh, prefit=True)
 select_X_train = selection.transform(X_train)
 	
selection_model = selection_model
selection_model.fit(select_X_train, y_train)
 	
select_X_test = selection.transform(X_test)
 y_pred = selection_model.predict_proba(select_X_test)[:,1]
predictions = y_pred
auc = calc_roc_auc(y_test, predictions)
 print("Thresh=%.3f, n=%d, AUC: %.2f%%" %(thresh, select_X_train.shape[1], auc))
confirmed = []
fatalities = []
for id, d, loc in tqdm(test[['ForecastId', 'Date', 'Location']].values):
c = to_exp(daily_log_confirmed.loc[daily_log_confirmed.Location == loc, d].values[0])
 f = to_exp(daily_log_deaths.loc[daily_log_deaths.Location == loc, d].values[0])
 confirmed.append(c)
 fatalities.append(f )<prepare_output>
sfm = feature_selection.SelectFromModel(xgbc, threshold=0.023, prefit= True)
 sfm_train= sfm.transform(train_merge)
 n_features = sfm_train.shape[1]
print(n_features )
my_submission = test.copy()
my_submission['ConfirmedCases'] = confirmed
my_submission['Fatalities'] = fatalities
my_submission.shape
my_submission.head() 
<save_to_csv>
xgb_clf= xgb.XGBClassifier(seed= 42)
 xgb_clf.fit(sfm_train, labels )
my_submission[[
'ForecastId', 'ConfirmedCases', 'Fatalities'
]].to_csv('submission.csv', index=False)
 print(DECAY)
 my_submission.head()
my_submission.tail()
my_submission.shape<train_model>
sfm_test = sfm.transform(test_merge)
 predictions_xgb = xgb_clf.predict_proba(sfm_test)[:,1]
end = dt.datetime.now()
print('Finished', end,(end - start ).seconds, 's' )<import_modules>
X_train= train_split
X_test= val_split
y_train= label_train_split
y_test= label_val_split
model= xgb.XGBClassifier(seed= 42)
 eval_set = [(X_train, y_train),(X_test, y_test)]
model.fit(X_train, y_train, eval_metric="auc", eval_set=eval_set, verbose=True )
print("Read in libraries")
<load_from_csv>
results = model.evals_result()
print(results )
print("read in train file")
 df=pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/train.csv",
usecols=['Province_State','Country_Region','Date','ConfirmedCases','Fatalities'])
<drop_column>
eval_set = [(X_test, y_test)]
model.fit(X_train, y_train, eval_metric=["auc"], eval_set=eval_set, early_stopping_rounds=10)
 results = model.evals_result()
print(results )
print("fill blanks and add region for counting")
 df.fillna(' ',inplace=True)
 df['Lat']=df['Province_State']+df['Country_Region']
df.drop('Province_State',axis=1,inplace=True)
 df.drop('Country_Region',axis=1,inplace=True)
<load_from_csv>
def report(results, n_top=3):
for i in range(1, n_top + 1):
candidates = np.flatnonzero(results['rank_test_score'] == i)
 for candidate in candidates:
print("Model with rank: {0}".format(i))
print("Mean validation score: {0:.3f}(std: {1:.3f})".format(
 results['mean_test_score'][candidate],
results['std_test_score'][candidate]))
print("Parameters: {0}".format(results['params'][candidate]))
print("" )
countries_list=df.Lat.unique()
df1=[]
for i in countries_list:
df1.append(df[df['Lat']==i])
 print("we have "+ str(len(df1)) +" regions in our dataset")
 test=pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/test.csv" )<choose_model_class>
n_estimators_dist= sps.randint(1, 300)
 learning_rate_dist = [0.01, 0.02, 0.05, 0.1, 0.2, 0.3]
submit_confirmed=[]
submit_fatal=[]
for i in df1:
data = i.ConfirmedCases.astype('int32' ).tolist()
try:
model = SARIMAX(data, order=(1,1,0), seasonal_order=(1,1,0,12),measurement_error=True)
 model_fit = model.fit(disp=False)
 predicted = model_fit.predict(len(data), len(data)+34)
 new=np.concatenate(( np.array(data),np.array([int(num)for num in predicted])) ,axis=0)
 submit_confirmed.extend(list(new[-43:]))
except:
submit_confirmed.extend(list(data[-10:-1]))
for j in range(34):
submit_confirmed.append(data[-1]*2)
 data = i.Fatalities.astype('int32' ).tolist()
try:
model = SARIMAX(data, order=(1,1,0), seasonal_order=(1,1,0,12),measurement_error=True)
 model_fit = model.fit(disp=False)
 predicted = model_fit.predict(len(data), len(data)+34)
 new=np.concatenate(( np.array(data),np.array([int(num)for num in predicted])) ,axis=0)
 submit_fatal.extend(list(new[-43:]))
except:
submit_fatal.extend(list(data[-10:-1]))
for j in range(34):
submit_fatal.append(data[-1]*2)
<data_type_conversions>
param_dist = dict(learning_rate= learning_rate_dist, n_estimators= n_estimators_dist)
 n_iter_search = 20
random_search = model_selection.RandomizedSearchCV(model, param_distributions=param_dist,
n_iter=n_iter_search, scoring= 'roc_auc')
 start = time()
random_search.fit(X_train, y_train)
 print("RandomizedSearchCV took %.2f seconds for %d candidates"
" parameter settings." %(( time() - start), n_iter_search))
report(random_search.cv_results_ )
<merge><EOS> 
sample_sub['WnvPresent'] = predictions_xgb
sample_sub.to_csv('sub_xgb.csv', index=False)