Kaggle competition: Bosch Production Line Performance
<SOS> metric: MatthewsCorrelationCoefficient Kaggle data source: jigsaw-toxic-severity-rating<groupby>
import gc
import numpy as np
import pandas as pd
import xgboost as xgb
from sklearn.metrics import matthews_corrcoef
from operator import itemgetter
df4[df4['Country_Region']=='Turkey']<merge>
def get_station_ohe() :
directory = '.. /input/'
trainfile = 'train_date.csv'
testfile = 'test_date.csv'
features = None
subset = None
train_date_part = pd.read_csv(directory + trainfile, nrows=10000)
 date_cols = train_date_part.drop('Id', axis=1 ).count().reset_index().sort_values(by=0, ascending=False)
 date_cols['station'] = date_cols['index'].apply(lambda s: s.split('_')[1])
 date_cols = date_cols.drop_duplicates('station', keep='first')['index'].tolist()
stations = list([f.split('_')[1] for f in date_cols ])
 stations = sorted(stations,key= lambda x: int(x[1:]))
for i, chunk in enumerate(pd.read_csv(directory + trainfile,
usecols=['Id'] + date_cols,
chunksize=50000,
low_memory=False)) :
if features is None:
features = list(chunk.columns)
 features.remove('Id')
 chunk.columns = ['Id'] + stations
chunk['start_station'] = -1
chunk['end_station'] = -1
for s in stations:
chunk[s] = 1 *(chunk[s] >= 0)
 id_not_null = chunk[chunk[s] == 1].Id
chunk.loc[(chunk['start_station']== -1)&(chunk.Id.isin(id_not_null)) ,'start_station'] = int(s[1:])
 chunk.loc[chunk.Id.isin(id_not_null),'end_station'] = int(s[1:])
 subset = pd.concat([subset, chunk])
 del chunk
gc.collect()
for i, chunk in enumerate(pd.read_csv(directory + testfile,
usecols=['Id'] + date_cols,
chunksize=50000,
low_memory=False)) :
chunk.columns = ['Id'] + stations
chunk['start_station'] = -1
chunk['end_station'] = -1
for s in stations:
chunk[s] = 1 *(chunk[s] >= 0)
 id_not_null = chunk[chunk[s] == 1].Id
chunk.loc[(chunk['start_station']== -1)&(chunk.Id.isin(id_not_null)) ,'start_station'] = int(s[1:])
 chunk.loc[chunk.Id.isin(id_not_null),'end_station'] = int(s[1:])
 subset = pd.concat([subset, chunk])
 del chunk
gc.collect()
return subset,stations,date_cols
<feature_engineering>
station_ohe,stations,date_cols = get_station_ohe()
if Optimistic:
df['Check'] = df["Date"].max() + timedelta(days=7)
 else:
df['Check'] = df["Date"].max() + timedelta(days=14)
 df['Date_Max'] = np.where(df['Date_Max']>df["Date"].max() - timedelta(days=1),df["Check"],df['Date_Max'])
<merge>
station_ohe['path_len'] = station_ohe[stations].sum(axis=1 )
test_df = preprocess(pd.read_csv(test_file),"Date")
 df = df.merge(test_df,how='outer', on =['Province_State','Country_Region','Date'])
 df=df.sort_values(['Province_State','Country_Region','Date'])
 df['Date_Max']=df.groupby(loc_group)['Date_Max'].fillna(method='ffill')
 if Optimistic:
df['Death_Date_Max'] = df['Date_Max']+ timedelta(days=5)
 else:
df['Death_Date_Max'] = df['Date_Max']+ timedelta(days=10 )<merge>
def get_date_features() :
directory = '.. /input/'
trainfile = 'train_date.csv'
for i, chunk in enumerate(pd.read_csv(directory + trainfile,
chunksize=1,
low_memory=False)) :
features = list(chunk.columns)
 del chunk
break
seen = np.zeros(52)
 rv = []
for f in features:
if f == 'Id' or 'S24' in f or 'S25' in f:
rv.append(f)
 continue
station = int(f.split('_')[1][1:])
 if seen[station]:
continue
seen[station] = 1
rv.append(f)
 return rv
usefuldatefeatures = get_date_features()
df = df.merge(df3,how='left',on=['Province_State','Country_Region'])
 df = df.merge(df4,how='left',on=['Province_State','Country_Region'] )<feature_engineering>
def create_new_feats() :
directory = '.. /input/'
trainfile = 'train_date.csv'
testfile = 'test_date.csv'
features = None
subset = None
for i, chunk in enumerate(pd.read_csv(directory + trainfile,
usecols=usefuldatefeatures,
chunksize=50000,
low_memory=False)) :
if features is None:
features = list(chunk.columns)
 features.remove('Id')
 week_duration = 1679
df_mindate_chunk = chunk[['Id']].copy()
df_mindate_chunk['mindate'] = chunk[features].min(axis=1 ).values
df_mindate_chunk['maxdate'] = chunk[features].max(axis=1 ).values
df_mindate_chunk['duration'] = df_mindate_chunk['maxdate'] - df_mindate_chunk['mindate']
df_mindate_chunk['part_week'] =(( df_mindate_chunk['mindate'].values * 100)% week_duration ).astype(np.int64)
 df_mindate_chunk['min_time_station'] = chunk[features].idxmin(axis = 1 ).apply(lambda s: int(s.split('_')[1][1:])if s is not np.nan else -1)
 df_mindate_chunk['max_time_station'] = chunk[features].idxmax(axis = 1 ).apply(lambda s: int(s.split('_')[1][1:])if s is not np.nan else -1)
 if subset is None:
subset = df_mindate_chunk.copy()
else:
subset = pd.concat([subset, df_mindate_chunk])
 del chunk
gc.collect()
for i, chunk in enumerate(pd.read_csv(directory + testfile,
usecols=usefuldatefeatures,
chunksize=50000,
low_memory=False)) :
df_mindate_chunk = chunk[['Id']].copy()
df_mindate_chunk['mindate'] = chunk[features].min(axis=1 ).values
df_mindate_chunk['maxdate'] = chunk[features].max(axis=1 ).values
df_mindate_chunk['duration'] = df_mindate_chunk['maxdate'] - df_mindate_chunk['mindate']
df_mindate_chunk['part_week'] =(( df_mindate_chunk['mindate'].values * 100)% week_duration ).astype(np.int64)
 df_mindate_chunk['min_time_station'] = chunk[features].idxmin(axis = 1 ).apply(lambda s: int(s.split('_')[1][1:])if s is not np.nan else -1)
 df_mindate_chunk['max_time_station'] = chunk[features].idxmax(axis = 1 ).apply(lambda s: int(s.split('_')[1][1:])if s is not np.nan else -1)
 subset = pd.concat([subset, df_mindate_chunk])
 del chunk
gc.collect()
return subset
df['Coeff']=df.groupby(loc_group)['Coeff'].fillna(method='ffill')
 df['Coeff'] = np.where(df['Coeff'].isna() ,0,df['Coeff'])
 df['Coeff'] = np.clip(df['Coeff'],0.05,0.75)
 df['Death_Coeff']=df.groupby(loc_group)['Death_Coeff'].fillna(method='ffill')
 df['Death_Coeff'] = np.where(df['Death_Coeff'].isna() ,0,df['Death_Coeff'])
 df['Death_Coeff'] = np.clip(df['Death_Coeff'],0.05,0.75)
 df['Death_Coeff'] = np.where(df['Death_Coeff']>df['Coeff'],df['Coeff'],df['Death_Coeff'])
 df['Coeff'] = np.where(df.Date <= df['Date_Max'],df['Coeff'],-0.7*df['Coeff'])
 df['Death_Coeff'] = np.where(df.Date <= df['Death_Date_Max'],df['Death_Coeff'],-0.7*df['Death_Coeff'] )<define_variables>
new_features = create_new_feats()
list1 =[]
list2=[]
pred = 0
change = 0
pred_death = 0
change_death = 0
for index, row in df.iterrows() :
if(row['Date']<=TRAIN_LAST)|(row['Date']==TRAIN_FIRST):
change =(1+row['Coeff'])*row['rolling_ConfirmedCases']
pred = row['ConfirmedCases']-change
change_death =(1+row['Death_Coeff'])*row['rolling_Fatalities']
pred_death = row['Fatalities']-change_death
else:
change = change*(1+row['Coeff'])
 change_death = change_death*(1+row['Death_Coeff'])
 if row['Date']==TEST_END:
change =(1+row['Coeff'])*row['rolling_ConfirmedCases']
pred = row['ConfirmedCases']-change
change_death =(1+row['Death_Coeff'])*row['rolling_Fatalities']
pred_death = row['Fatalities']-change_death
if change<0:
change =0
pred = pred+change
list1.append(pred)
 if change_death<0:
change_death =0
pred_death = pred_death+change_death
if pred_death>pred*0.1:
pred_death = pred*0.1
list2.append(pred_death)
 df['pred_CC']= list1
df['pred_Fatalities']= list2<save_to_csv>
new_features.sort_values(by=['mindate', 'Id'], inplace=True )
df.to_csv("df2.csv" )<save_to_csv>
new_features['mindate_id_diff'] = new_features.Id.diff()
df.groupby('Date')[['ConfirmedCases','Fatalities','pred_CC','pred_Fatalities']].sum().to_csv('sum.csv' )<filter>
midr = np.full_like(new_features.mindate_id_diff.values, np.nan )
df[df['Province_State']=='New York'][['Date','ConfirmedCases','Fatalities','pred_CC','pred_Fatalities','Coeff','Death_Coeff','rolling_ConfirmedCases','Date_Max','Death_Date_Max']]<save_to_csv>
midr[0:-1] = -new_features.mindate_id_diff.values[1:]
df['Error_CC']= np.where(df['ConfirmedCases']==0,0,(abs(( df['pred_CC']-df['ConfirmedCases'])/df['ConfirmedCases'])))
 df['Error_Death']= np.where(df['Fatalities']==0,0,abs(( df['pred_Fatalities']-df['Fatalities'])/df['Fatalities']))
df.groupby('Date')[['Error_CC','Error_Death']].sum().to_csv('error.csv' )<filter>
new_features['mindate_id_diff_reverse'] = midr
df[df['Country_Region']=='Spain'][['Date','ConfirmedCases','Fatalities','pred_CC','pred_Fatalities','Coeff','Death_Coeff','rolling_ConfirmedCases','Date_Max','Death_Date_Max']]<load_from_csv>
def mcc(tp, tn, fp, fn):
sup = tp * tn - fp * fn
inf =(tp + fp)*(tp + fn)*(tn + fp)*(tn + fn)
 if inf == 0:
return 0
else:
return sup / np.sqrt(inf )
sub_df = pd.read_csv(sub_file)
<merge>
def eval_mcc(y_true, y_prob, show=False):
idx = np.argsort(y_prob)
 y_true_sort = y_true[idx]
n = y_true.shape[0]
nump = 1.0 * np.sum(y_true)
 numn = n - nump
tp = nump
tn = 0.0
fp = numn
fn = 0.0
best_mcc = 0.0
best_id = -1
mccs = np.zeros(n)
 for i in range(n):
if y_true_sort[i] == 1:
tp -= 1.0
fn += 1.0
else:
fp -= 1.0
tn += 1.0
new_mcc = mcc(tp, tn, fp, fn)
 mccs[i] = new_mcc
if new_mcc >= best_mcc:
best_mcc = new_mcc
best_id = i
if show:
best_proba = y_prob[idx[best_id]]
y_pred =(y_prob > best_proba ).astype(int)
 return best_proba, best_mcc, y_pred
else:
return best_mcc
sub_df = sub_df.merge(df[['ForecastId','pred_CC','pred_Fatalities']],how='left',on='ForecastId' )<data_type_conversions>
def mcc_eval(y_prob, dtrain):
y_true = dtrain.get_label()
best_mcc = eval_mcc(y_true, y_prob)
 return 'MCC', best_mcc
sub_df['ConfirmedCases']= sub_df['pred_CC'].astype('int')
 sub_df['Fatalities']= sub_df['pred_Fatalities'].astype('int')
 sub_df = sub_df.drop(['pred_CC','pred_Fatalities'],axis=1 )<save_to_csv>
directory = '.. /input/'
trainfiles = ['train_date.csv',
'train_numeric.csv']
testfiles = ['test_date.csv',
'test_numeric.csv']
sub_df.to_csv("submission.csv", index=False )<set_options>
num_feats = ['Id',
'L3_S33_F3855',
'L3_S32_F3850',
'L3_S33_F3865',
'L1_S24_F1581',
'L3_S38_F3952',
'L1_S24_F1672',
'L1_S24_F1632',
'L1_S24_F1846',
'L1_S24_F1844',
'L1_S24_F1609',
'L1_S24_F1667',
'L1_S24_F1842',
'L0_S13_F356',
'L3_S29_F3342',
'L3_S29_F3407',
'L3_S34_F3876',
'L0_S11_F302',
'L3_S29_F3461',
'L3_S30_F3494',
'L0_S3_F100',
'L0_S1_F28',
'L0_S6_F122',
'L0_S0_F0',
'L0_S0_F20',
'L3_S30_F3704',
'Response']
pd.set_option('max_columns', 500)
 pd.set_option('max_rows', 500)
 pd.options.display.float_format = '{:.2f}'.format<load_from_csv>
cols = [['Id']+date_cols,num_feats]
train = pd.read_csv(".. /input/covid19-global-forecasting-week-2/train.csv")
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-2/test.csv")
 tt = pd.concat([train, test], sort=False)
 tt = train.merge(test, on=['Province_State','Country_Region','Date'], how='outer')
 def name_place(x):
try:
x_new = x['Country_Region'] + "_" + x['Province_State']
except:
x_new = x['Country_Region']
return x_new
tt['Place'] = tt.apply(lambda x: name_place(x), axis=1)
 tt['Date'] = pd.to_datetime(tt['Date'])
 tt['doy'] = tt['Date'].dt.dayofyear
tt['dow'] = tt['Date'].dt.dayofweek
tt['hasProvidence'] = ~tt['Province_State'].isna()
country_meta = pd.read_csv('.. /input/covid19-forecasting-metadata/region_metadata.csv')
 tt = tt.merge(country_meta, how='left')
 country_date_meta = pd.read_csv('.. /input/covid19-forecasting-metadata/region_date_metadata.csv')
 tt['HasFatality'] = tt.groupby('Place')['Fatalities'].transform(lambda x: x.max() > 0)
 tt['HasCases'] = tt.groupby('Place')['ConfirmedCases'].transform(lambda x: x.max() > 0)
 first_case_date = tt.query('ConfirmedCases >= 1' ).groupby('Place')['Date'].min().to_dict()
ten_case_date = tt.query('ConfirmedCases >= 10' ).groupby('Place')['Date'].min().to_dict()
hundred_case_date = tt.query('ConfirmedCases >= 100' ).groupby('Place')['Date'].min().to_dict()
first_fatal_date = tt.query('Fatalities >= 1' ).groupby('Place')['Date'].min().to_dict()
ten_fatal_date = tt.query('Fatalities >= 10' ).groupby('Place')['Date'].min().to_dict()
hundred_fatal_date = tt.query('Fatalities >= 100' ).groupby('Place')['Date'].min().to_dict()
tt['First_Case_Date'] = tt['Place'].map(first_case_date)
 tt['Ten_Case_Date'] = tt['Place'].map(ten_case_date)
 tt['Hundred_Case_Date'] = tt['Place'].map(hundred_case_date)
 tt['First_Fatal_Date'] = tt['Place'].map(first_fatal_date)
 tt['Ten_Fatal_Date'] = tt['Place'].map(ten_fatal_date)
 tt['Hundred_Fatal_Date'] = tt['Place'].map(hundred_fatal_date)
 tt['Days_Since_First_Case'] =(tt['Date'] - tt['First_Case_Date'] ).dt.days
tt['Days_Since_Ten_Cases'] =(tt['Date'] - tt['Ten_Case_Date'] ).dt.days
tt['Days_Since_Hundred_Cases'] =(tt['Date'] - tt['Hundred_Case_Date'] ).dt.days
tt['Days_Since_First_Fatal'] =(tt['Date'] - tt['First_Fatal_Date'] ).dt.days
tt['Days_Since_Ten_Fatal'] =(tt['Date'] - tt['Ten_Fatal_Date'] ).dt.days
tt['Days_Since_Hundred_Fatal'] =(tt['Date'] - tt['Hundred_Fatal_Date'] ).dt.days
smoking = pd.read_csv(".. /input/smokingstats/share-of-adults-who-smoke.csv")
 smoking = smoking.rename(columns={'Smoking prevalence, total(ages 15+ )(% of adults)': 'Smoking_Rate'})
 smoking_dict = smoking.groupby('Entity')['Year'].max().to_dict()
smoking['LastYear'] = smoking['Entity'].map(smoking_dict)
 smoking = smoking.query('Year == LastYear' ).reset_index()
smoking['Entity'] = smoking['Entity'].str.replace('United States', 'US')
 tt = tt.merge(smoking[['Entity','Smoking_Rate']],
left_on='Country_Region',
right_on='Entity',
how='left',
validate='m:1')\ 
.drop('Entity', axis=1)
 country_info = pd.read_csv('.. /input/countryinfo/covid19countryinfo.csv')
 tt = tt.merge(country_info, left_on=['Country_Region','Province_State'],
right_on=['country','region'],
how='left',
validate='m:1')
 us_state_info = pd.read_html('https://simple.wikipedia.org/wiki/List_of_U.S._states_by_population')[0] \
[['State','Population estimate, July 1, 2019[2]']] \ 
.rename(columns={'Population estimate, July 1, 2019[2]' : 'Population'})
 tt = tt.merge(us_state_info[['State','Population']],
left_on='Province_State',
right_on='State',
how='left')
 tt['pop'] = pd.to_numeric(tt['pop'].str.replace(',',''))
tt['pop'] = tt['pop'].fillna(tt['Population'])
 tt['pop'] = pd.to_numeric(tt['pop'])
 tt['pop_diff'] = tt['pop'] - tt['Population']
tt['Population_final'] = tt['Population']
tt.loc[~tt['hasProvidence'], 'Population_final'] = tt.loc[~tt['hasProvidence']]['pop']
tt['Confirmed_Cases_Diff'] = tt.groupby('Place')['ConfirmedCases'].diff()
tt['Fatailities_Diff'] = tt.groupby('Place')['Fatalities'].diff()
max_date = tt.dropna(subset=['ConfirmedCases'])['Date'].max()
tt['gdp2019'] = pd.to_numeric(tt['gdp2019'].str.replace(',',''))<define_variables>
traindata = None
testdata = None
pop_dict = {'Angola': int(29.78 * 10**6),
'Australia_Australian Capital Territory': 423_800,
'Australia_New South Wales': int(7.544 * 10**6),
'Australia_Northern Territory': 244_300,
'Australia_Queensland' : int(5.071 * 10**6),
'Australia_South Australia' : int(1.677 * 10**6),
'Australia_Tasmania': 515_000,
'Australia_Victoria': int(6.359 * 10**6),
'Australia_Western Australia': int(2.589 * 10**6),
'Brazil': int(209.3 * 10**6),
'Canada_Alberta' : int(4.371 * 10**6),
'Canada_British Columbia' : int(5.071 * 10**6),
'Canada_Manitoba' : int(1.369 * 10**6),
'Canada_New Brunswick' : 776_827,
'Canada_Newfoundland and Labrador' : 521_542,
'Canada_Nova Scotia' : 971_395,
'Canada_Ontario' : int(14.57 * 10**6),
'Canada_Prince Edward Island' : 156_947,
'Canada_Quebec' : int(8.485 * 10**6),
'Canada_Saskatchewan': int(1.174 * 10**6),
'China_Anhui': int(62 * 10**6),
'China_Beijing': int(21.54 * 10**6),
'China_Chongqing': int(30.48 * 10**6),
'China_Fujian' : int(38.56 * 10**6),
'China_Gansu' : int(25.58 * 10**6),
'China_Guangdong' : int(113.46 * 10**6),
'China_Guangxi' : int(48.38 * 10**6),
'China_Guizhou' : int(34.75 * 10**6),
'China_Hainan' : int(9.258 * 10**6),
'China_Hebei' : int(74.7 * 10**6),
'China_Heilongjiang' : int(38.31 * 10**6),
'China_Henan' : int(94 * 10**6),
'China_Hong Kong' : int(7.392 * 10**6),
'China_Hubei' : int(58.5 * 10**6),
'China_Hunan' : int(67.37 * 10**6),
'China_Inner Mongolia' : int(24.71 * 10**6),
'China_Jiangsu' : int(80.4 * 10**6),
'China_Jiangxi' : int(45.2 * 10**6),
'China_Jilin' : int(27.3 * 10**6),
'China_Liaoning' : int(43.9 * 10**6),
'China_Macau' : 622_567,
'China_Ningxia' : int(6.301 * 10**6),
'China_Qinghai' : int(5.627 * 10**6),
'China_Shaanxi' : int(37.33 * 10**6),
'China_Shandong' : int(92.48 * 10**6),
'China_Shanghai' : int(24.28 * 10**6),
'China_Shanxi' : int(36.5 * 10**6),
'China_Sichuan' : int(81.1 * 10**6),
'China_Tianjin' : int(15 * 10**6),
'China_Tibet' : int(3.18 * 10**6),
'China_Xinjiang' : int(21.81 * 10**6),
'China_Yunnan' : int(45.97 * 10**6),
'China_Zhejiang' : int(57.37 * 10**6),
'Denmark_Faroe Islands' : 51_783,
'Denmark_Greenland' : 56_171,
'France_French Guiana' : 290_691,
'France_French Polynesia' : 283_007,
'France_Guadeloupe' : 395_700,
'France_Martinique' : 376_480,
'France_Mayotte' : 270_372,
'France_New Caledonia' : 99_926,
'France_Reunion' : 859_959,
'France_Saint Barthelemy' : 9_131,
'France_St Martin' : 32_125,
'Netherlands_Aruba' : 105_264,
'Netherlands_Curacao' : 161_014,
'Netherlands_Sint Maarten' : 41_109,
'Papua New Guinea' : int(8.251 * 10**6),
'US_Guam' : 164_229,
'US_Virgin Islands' : 107_268,
'United Kingdom_Bermuda' : 65_441,
'United Kingdom_Cayman Islands' : 61_559,
'United Kingdom_Channel Islands' : 170_499,
'United Kingdom_Gibraltar' : 34_571,
'United Kingdom_Isle of Man' : 84_287,
'United Kingdom_Montserrat' : 4_922
}
tt['Population_final'] = tt['Population_final'].fillna(tt['Place'].map(pop_dict))<feature_engineering>
for i, f in enumerate(trainfiles):
print(f)
 subset = None
for i, chunk in enumerate(pd.read_csv(directory + f,
usecols=cols[i],
chunksize=50000,
low_memory=False)) :
if subset is None:
subset = chunk.copy()
else:
subset = pd.concat([subset, chunk])
 del chunk
gc.collect()
if traindata is None:
traindata = subset.copy()
else:
traindata = pd.merge(traindata, subset.copy() , on="Id")
 del subset
gc.collect()
tt.loc[tt['Place'] == 'Diamond Princess', 'Population final'] = 2_670<feature_engineering>
del cols[1][-1]
tt['ConfirmedCases_Log'] = tt['ConfirmedCases'].apply(np.log1p)
 tt['Fatalities_Log'] = tt['Fatalities'].apply(np.log1p )<feature_engineering>
for i, f in enumerate(testfiles):
print(f)
 subset = None
for i, chunk in enumerate(pd.read_csv(directory + f,
usecols=cols[i],
chunksize=50000,
low_memory=False)) :
if subset is None:
subset = chunk.copy()
else:
subset = pd.concat([subset, chunk])
 del chunk
gc.collect()
if testdata is None:
testdata = subset.copy()
else:
testdata = pd.merge(testdata, subset.copy() , on="Id")
 del subset
gc.collect()
tt['Population_final'] = tt['Population_final'].astype('int')
 tt['Cases_Per_100kPop'] =(tt['ConfirmedCases'] / tt['Population_final'])* 100000
tt['Fatalities_Per_100kPop'] =(tt['Fatalities'] / tt['Population_final'])* 100000
tt['Cases_Percent_Pop'] =(( tt['ConfirmedCases'] / tt['Population_final'])* 100)
 tt['Fatalities_Percent_Pop'] =(( tt['Fatalities'] / tt['Population_final'])* 100)
 tt['Cases_Log_Percent_Pop'] =(( tt['ConfirmedCases'] / tt['Population_final'])* 100 ).apply(np.log1p)
 tt['Fatalities_Log_Percent_Pop'] =(( tt['Fatalities'] / tt['Population_final'])* 100 ).apply(np.log1p)
 tt['Max_Confirmed_Cases'] = tt.groupby('Place')['ConfirmedCases'].transform(max)
 tt['Max_Fatalities'] = tt.groupby('Place')['Fatalities'].transform(max)
 tt['Max_Cases_Per_100kPop'] = tt.groupby('Place')['Cases_Per_100kPop'].transform(max)
 tt['Max_Fatalities_Per_100kPop'] = tt.groupby('Place')['Fatalities_Per_100kPop'].transform(max )<remove_duplicates>
del midr
gc.collect()
tt.query('Date == @max_date')[['Place','Max_Cases_Per_100kPop',
'Max_Fatalities_Per_100kPop','Max_Confirmed_Cases',
'Population_final',
'Days_Since_First_Case',
'Confirmed_Cases_Diff']] \ 
.drop_duplicates() \ 
.sort_values('Max_Cases_Per_100kPop', ascending=False )<categorify>
traindata = traindata.merge(new_features, on='Id')
 traindata = traindata.merge(station_ohe, on='Id')
 testdata = testdata.merge(new_features, on='Id')
 testdata = testdata.merge(station_ohe, on='Id' )
tt['Past_7Days_ConfirmedCases_Std'] = tt['Place'].map(tt.dropna(subset=['ConfirmedCases'] ).query('Date >= 20200324' ).groupby('Place')['ConfirmedCases'].std().to_dict())
 tt['Past_7Days_Fatalities_Std'] = tt['Place'].map(tt.dropna(subset=['ConfirmedCases'] ).query('Date >= 20200324' ).groupby('Place')['Fatalities'].std().to_dict())
 tt['Past_7Days_ConfirmedCases_Min'] = tt['Place'].map(tt.dropna(subset=['ConfirmedCases'] ).query('Date >= 20200324' ).groupby('Place')['ConfirmedCases'].min().to_dict())
 tt['Past_7Days_Fatalities_Min'] = tt['Place'].map(tt.dropna(subset=['Fatalities'] ).query('Date >= 20200324' ).groupby('Place')['Fatalities'].min().to_dict())
 tt['Past_7Days_ConfirmedCases_Max'] = tt['Place'].map(tt.dropna(subset=['ConfirmedCases'] ).query('Date >= 20200324' ).groupby('Place')['ConfirmedCases'].max().to_dict())
 tt['Past_7Days_Fatalities_Max'] = tt['Place'].map(tt.dropna(subset=['Fatalities'] ).query('Date >= 20200324' ).groupby('Place')['Fatalities'].max().to_dict())
 tt['Past_7Days_Confirmed_Change_of_Total'] =(tt['Past_7Days_ConfirmedCases_Max'] - tt['Past_7Days_ConfirmedCases_Min'])/(tt['Past_7Days_ConfirmedCases_Max'])
 tt['Past_7Days_Fatalities_Change_of_Total'] =(tt['Past_7Days_Fatalities_Max'] - tt['Past_7Days_Fatalities_Min'])/(tt['Past_7Days_Fatalities_Max'] )<categorify>
del new_features
del station_ohe
gc.collect()
tt['Past_21Days_ConfirmedCases_Std'] = tt['Place'].map(tt.dropna(subset=['ConfirmedCases'] ).query('Date >= 20200310' ).groupby('Place')['ConfirmedCases'].std().to_dict())
 tt['Past_21Days_Fatalities_Std'] = tt['Place'].map(tt.dropna(subset=['ConfirmedCases'] ).query('Date >= 20200310' ).groupby('Place')['Fatalities'].std().to_dict())
 tt['Past_21Days_ConfirmedCases_Min'] = tt['Place'].map(tt.dropna(subset=['ConfirmedCases'] ).query('Date >= 20200310' ).groupby('Place')['ConfirmedCases'].min().to_dict())
 tt['Past_21Days_Fatalities_Min'] = tt['Place'].map(tt.dropna(subset=['Fatalities'] ).query('Date >= 20200324' ).groupby('Place')['Fatalities'].min().to_dict())
 tt['Past_21Days_ConfirmedCases_Max'] = tt['Place'].map(tt.dropna(subset=['ConfirmedCases'] ).query('Date >= 20200310' ).groupby('Place')['ConfirmedCases'].max().to_dict())
 tt['Past_21Days_Fatalities_Max'] = tt['Place'].map(tt.dropna(subset=['Fatalities'] ).query('Date >= 20200310' ).groupby('Place')['Fatalities'].max().to_dict())
 tt['Past_21Days_Confirmed_Change_of_Total'] =(tt['Past_21Days_ConfirmedCases_Max'] - tt['Past_21Days_ConfirmedCases_Min'])/(tt['Past_21Days_ConfirmedCases_Max'])
 tt['Past_21Days_Fatalities_Change_of_Total'] =(tt['Past_21Days_Fatalities_Max'] - tt['Past_21Days_Fatalities_Min'])/(tt['Past_21Days_Fatalities_Max'])
 tt['Past_7Days_Fatalities_Change_of_Total'] = tt['Past_7Days_Fatalities_Change_of_Total'].fillna(0)
 tt['Past_21Days_Fatalities_Change_of_Total'] = tt['Past_21Days_Fatalities_Change_of_Total'].fillna(0 )<categorify>
testdata['Response'] = 0
tt['Date_7Days_Since_First_Case'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Case'] == 7] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_14Days_Since_First_Case'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Case'] == 14] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_21Days_Since_First_Case'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Case'] == 21] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_28Days_Since_First_Case'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Case'] == 28] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_35Days_Since_First_Case'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Case'] == 35] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_60Days_Since_First_Case'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Case'] == 60] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_7Days_Since_Ten_Cases'] = tt['Place'].map(tt.loc[tt['Days_Since_Ten_Cases'] == 7] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_14Days_Since_Ten_Cases'] = tt['Place'].map(tt.loc[tt['Days_Since_Ten_Cases'] == 14] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_21Days_Since_Ten_Cases'] = tt['Place'].map(tt.loc[tt['Days_Since_Ten_Cases'] == 21] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_28Days_Since_Ten_Cases'] = tt['Place'].map(tt.loc[tt['Days_Since_Ten_Cases'] == 28] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_35Days_Since_Ten_Cases'] = tt['Place'].map(tt.loc[tt['Days_Since_Ten_Cases'] == 35] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_60Days_Since_Ten_Cases'] = tt['Place'].map(tt.loc[tt['Days_Since_Ten_Cases'] == 60] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_7Days_Since_First_Fatal'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Fatal'] == 7] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_14Days_Since_First_Fatal'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Fatal'] == 14] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_21Days_Since_First_Fatal'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Fatal'] == 21] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_28Days_Since_First_Fatal'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Fatal'] == 28] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_35Days_Since_First_Fatal'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Fatal'] == 35] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_60Days_Since_First_Fatal'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Fatal'] == 60] \ 
.set_index('Place')['Date'] \ 
.to_dict() )<feature_engineering>
visibletraindata = traindata[::2]
tt['CC_7D_1stCase'] = tt.loc[tt['Date_7Days_Since_First_Case'] == tt['Date']]['ConfirmedCases']
tt['CC_14D_1stCase'] = tt.loc[tt['Date_14Days_Since_First_Case'] == tt['Date']]['ConfirmedCases']
tt['CC_21D_1stCase'] = tt.loc[tt['Date_21Days_Since_First_Case'] == tt['Date']]['ConfirmedCases']
tt['CC_28D_1stCase'] = tt.loc[tt['Date_28Days_Since_First_Case'] == tt['Date']]['ConfirmedCases']
tt['CC_35D_1stCase'] = tt.loc[tt['Date_35Days_Since_First_Case'] == tt['Date']]['ConfirmedCases']
tt['CC_60D_1stCase'] = tt.loc[tt['Date_60Days_Since_First_Case'] == tt['Date']]['ConfirmedCases']
tt['F_7D_1stCase'] = tt.loc[tt['Date_7Days_Since_First_Case'] == tt['Date']]['Fatalities']
tt['F_14D_1stCase'] = tt.loc[tt['Date_14Days_Since_First_Case'] == tt['Date']]['Fatalities']
tt['F_21D_1stCase'] = tt.loc[tt['Date_21Days_Since_First_Case'] == tt['Date']]['Fatalities']
tt['F_28D_1stCase'] = tt.loc[tt['Date_28Days_Since_First_Case'] == tt['Date']]['Fatalities']
tt['F_35D_1stCase'] = tt.loc[tt['Date_35Days_Since_First_Case'] == tt['Date']]['Fatalities']
tt['F_60D_1stCase'] = tt.loc[tt['Date_60Days_Since_First_Case'] == tt['Date']]['Fatalities']
tt['CC_7D_10Case'] = tt.loc[tt['Date_7Days_Since_Ten_Cases'] == tt['Date']]['ConfirmedCases']
tt['CC_14D_10Case'] = tt.loc[tt['Date_14Days_Since_Ten_Cases'] == tt['Date']]['ConfirmedCases']
tt['CC_21D_10Case'] = tt.loc[tt['Date_21Days_Since_Ten_Cases'] == tt['Date']]['ConfirmedCases']
tt['CC_28D_10Case'] = tt.loc[tt['Date_28Days_Since_Ten_Cases'] == tt['Date']]['ConfirmedCases']
tt['CC_35D_10Case'] = tt.loc[tt['Date_35Days_Since_Ten_Cases'] == tt['Date']]['ConfirmedCases']
tt['CC_60D_10Case'] = tt.loc[tt['Date_60Days_Since_Ten_Cases'] == tt['Date']]['ConfirmedCases']
tt['F_7D_10Case'] = tt.loc[tt['Date_7Days_Since_Ten_Cases'] == tt['Date']]['Fatalities']
tt['F_14D_10Case'] = tt.loc[tt['Date_14Days_Since_Ten_Cases'] == tt['Date']]['Fatalities']
tt['F_21D_10Case'] = tt.loc[tt['Date_21Days_Since_Ten_Cases'] == tt['Date']]['Fatalities']
tt['F_28D_10Case'] = tt.loc[tt['Date_28Days_Since_Ten_Cases'] == tt['Date']]['Fatalities']
tt['F_35D_10Case'] = tt.loc[tt['Date_35Days_Since_Ten_Cases'] == tt['Date']]['Fatalities']
tt['F_60D_10Case'] = tt.loc[tt['Date_60Days_Since_Ten_Cases'] == tt['Date']]['Fatalities']
tt['CC_7D_1Fatal'] = tt.loc[tt['Date_7Days_Since_First_Fatal'] == tt['Date']]['ConfirmedCases']
tt['CC_14D_1Fatal'] = tt.loc[tt['Date_14Days_Since_First_Fatal'] == tt['Date']]['ConfirmedCases']
tt['CC_21D_1Fatal'] = tt.loc[tt['Date_21Days_Since_First_Fatal'] == tt['Date']]['ConfirmedCases']
tt['CC_28D_1Fatal'] = tt.loc[tt['Date_28Days_Since_First_Fatal'] == tt['Date']]['ConfirmedCases']
tt['CC_35D_1Fatal'] = tt.loc[tt['Date_35Days_Since_First_Fatal'] == tt['Date']]['ConfirmedCases']
tt['CC_60D_1Fatal'] = tt.loc[tt['Date_60Days_Since_First_Fatal'] == tt['Date']]['ConfirmedCases']
tt['F_7D_1Fatal'] = tt.loc[tt['Date_7Days_Since_First_Fatal'] == tt['Date']]['Fatalities']
tt['F_14D_1Fatal'] = tt.loc[tt['Date_14Days_Since_First_Fatal'] == tt['Date']]['Fatalities']
tt['F_21D_1Fatal'] = tt.loc[tt['Date_21Days_Since_First_Fatal'] == tt['Date']]['Fatalities']
tt['F_28D_1Fatal'] = tt.loc[tt['Date_28Days_Since_First_Fatal'] == tt['Date']]['Fatalities']
tt['F_35D_1Fatal'] = tt.loc[tt['Date_35Days_Since_First_Fatal'] == tt['Date']]['Fatalities']
tt['F_60D_1Fatal'] = tt.loc[tt['Date_60Days_Since_First_Fatal'] == tt['Date']]['Fatalities']<remove_duplicates>
blindtraindata = traindata[1::2]
latest_summary_stats = tt.query('Date == @max_date')\
[['Country_Region',
'Place',
'Max_Cases_Per_100kPop',
'Max_Fatalities_Per_100kPop',
'Max_Confirmed_Cases',
'Population_final',
'Days_Since_First_Case',
'Days_Since_Ten_Cases']] \ 
.drop_duplicates()<import_modules>
del traindata
gc.collect()
from sklearn.linear_model import LinearRegression, ElasticNet<prepare_x_and_y>
def LeaveOneOut(data1, data2, columnName, useLOO=False):
grpOutcomes = data1.groupby(columnName)['Response'].mean().reset_index()
grpCount = data1.groupby(columnName)['Response'].count().reset_index()
grpOutcomes['cnt'] = grpCount.Response
if(useLOO):
grpOutcomes = grpOutcomes[grpOutcomes.cnt > 1]
grpOutcomes.drop('cnt', inplace=True, axis=1)
 outcomes = data2['Response'].values
x = pd.merge(data2[[columnName, 'Response']], grpOutcomes,
suffixes=('x_', ''),
how='left',
on=columnName,
left_index=True)['Response']
if(useLOO):
x =(( x*x.shape[0])-outcomes)/(x.shape[0]-1)
 return x.fillna(x.mean() )
for myplace in us_states:
try:
fig, axs = plt.subplots(1, 2, figsize=(15, 3))
dat = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','ConfirmedCases_Log']].dropna()
X = dat['Days_Since_Ten_Cases']
y = dat['ConfirmedCases_Log']
y = y.cummax()
dat_all = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','ConfirmedCases_Log']]
X_pred = dat_all['Days_Since_Ten_Cases']
en = ElasticNet()
en.fit(X.values.reshape(-1, 1), y.values)
 preds = en.predict(X_pred.values.reshape(-1, 1))
tt.loc[(tt['Place'] == myplace)&(tt['Days_Since_Ten_Cases'] >= 0), 'ConfirmedCases_Log_Pred1'] = preds
tt.loc[(tt['Place'] == myplace), 'ConfirmedCases_Pred1'] = tt['ConfirmedCases_Log_Pred1'].apply(np.expm1)
 pop_myplace = tt.query('Place == @myplace')['Population_final'].values[0]
tt.loc[(tt['Place'] == myplace)&(tt['ConfirmedCases_Pred1'] >(0.05 * pop_myplace)) , 'ConfirmedCases_Pred1'] =(0.05 * pop_myplace)
 ax = tt.query('Place == @myplace' ).set_index('Date')[['ConfirmedCases','ConfirmedCases_Pred1']].plot(figsize=(15, 5), title=myplace, ax=axs[0])
 dat = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','Fatalities_Log']].dropna()
if len(dat)< 5:
tt.loc[(tt['Place'] == myplace), 'Fatalities_Pred1'] = tt.loc[(tt['Place'] == myplace)]['ConfirmedCases_Pred1'] * 0.0001
elif tt.query('Place == @myplace')['Fatalities'].max() < 5:
tt.loc[(tt['Place'] == myplace), 'Fatalities_Pred1'] = tt.loc[(tt['Place'] == myplace)]['ConfirmedCases_Pred1'] * 0.0001
else:
X = dat['Days_Since_Ten_Cases']
y = dat['Fatalities_Log']
y = y.cummax()
dat_all = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','Fatalities_Log']]
X_pred = dat_all['Days_Since_Ten_Cases']
en = ElasticNet()
en.fit(X.values.reshape(-1, 1), y.values)
 preds = en.predict(X_pred.values.reshape(-1, 1))
tt.loc[(tt['Place'] == myplace)&(tt['Days_Since_Ten_Cases'] >= 0), 'Fatalities_Log_Pred1'] = preds
tt.loc[(tt['Place'] == myplace), 'Fatalities_Pred1'] = tt['Fatalities_Log_Pred1'].apply(np.expm1)
 pop_myplace = tt.query('Place == @myplace')['Population_final'].values[0]
tt.loc[(tt['Place'] == myplace)&(tt['Fatalities_Pred1'] >(0.0001 * pop_myplace)) , 'Fatalities_Pred1'] =(0.0001 * pop_myplace)
 ax = tt.query('Place == @myplace' ).set_index('Date')[['Fatalities','Fatalities_Pred1']].plot(figsize=(15, 5), title=myplace, ax=axs[1])
 plt.show()
except:
print(f'============= FAILED FOR {myplace} =============')
<drop_column>
for i in range(2):
for col in cols[i][1:]:
blindtraindata.loc[:, col] = LeaveOneOut(visibletraindata,
blindtraindata,
col, False ).values
testdata.loc[:, col] = LeaveOneOut(visibletraindata,
testdata, col, False ).values
myplace = 'US_Virgin Islands'
dat = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Place','Days_Since_Ten_Cases','ConfirmedCases']].dropna()<groupby>
num_rounds =52
params = {}
params['objective'] = "binary:logistic"
params['eta'] = 0.02105
params['max_depth'] = 28
params['colsample_bytree'] = 0.999
params['subsample'] = 0.999999
params['min_child_weight'] = 3
params['base_score'] = 0.0044
params['silent'] = True
params['eval_metric']='auc'
print('Fitting')
tt.loc[tt['Place'].isin(us_states)].groupby('Place')['Fatalities_Pred1'].max().sum()<feature_engineering>
trainpredictions = None
testpredictions = None
features = list(blindtraindata.columns)
 features.remove('Response')
 features.remove('Id')
 dvisibletrain = \
xgb.DMatrix(blindtraindata[features],
blindtraindata.Response,
silent=True)
 dtest = \
xgb.DMatrix(testdata[features],
silent=True)
 folds = 1
tt.loc[tt['Place'].isin(constant_fatal_places), 'ConfirmedCases_Pred1'] = tt.loc[tt['Place'].isin(constant_fatal_places)]['Place'].map(tt.loc[tt['Place'].isin(constant_fatal_places)].groupby('Place')['ConfirmedCases'].max())
 tt.loc[tt['Place'].isin(constant_fatal_places), 'Fatalities_Pred1'] = tt.loc[tt['Place'].isin(constant_fatal_places)]['Place'].map(tt.loc[tt['Place'].isin(constant_fatal_places)].groupby('Place')['Fatalities'].max() )<create_dataframe>
for i in range(folds):
print('Fold:', i)
 params['seed'] = i
watchlist = [(dvisibletrain, 'train'),(dvisibletrain, 'val')]
clf = xgb.train(params, dvisibletrain,
num_boost_round=num_rounds,
evals=watchlist,
early_stopping_rounds=20,
feval=mcc_eval,
maximize=True 
)
 limit = clf.best_iteration+1
predictions = \
clf.predict(dvisibletrain, ntree_limit=limit)
remaining_places = pd.DataFrame(tt.groupby('Place')['ConfirmedCases_Pred1'].max().isna() ).query('ConfirmedCases_Pred1' ).index.values
print(remaining_places)
 print(len(remaining_places))<prepare_x_and_y>
best_proba, best_mcc, y_pred = eval_mcc(dvisibletrain.Response,
predictions,
True)
 print(best_proba)
 print(best_mcc)
for myplace in remaining_places:
try:
fig, axs = plt.subplots(1, 2, figsize=(15, 3))
dat = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','ConfirmedCases_Log']].dropna()
X = dat['Days_Since_Ten_Cases']
y = dat['ConfirmedCases_Log']
y = y.cummax()
dat_all = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','ConfirmedCases_Log']]
X_pred = dat_all['Days_Since_Ten_Cases']
en = ElasticNet()
en.fit(X.values.reshape(-1, 1), y.values)
 preds = en.predict(X_pred.values.reshape(-1, 1))
tt.loc[(tt['Place'] == myplace)&(tt['Days_Since_Ten_Cases'] >= 0), 'ConfirmedCases_Log_Pred1'] = preds
tt.loc[(tt['Place'] == myplace), 'ConfirmedCases_Pred1'] = tt['ConfirmedCases_Log_Pred1'].apply(np.expm1)
 pop_myplace = tt.query('Place == @myplace')['Population_final'].values[0]
tt.loc[(tt['Place'] == myplace)&(tt['ConfirmedCases_Pred1'] >(0.05 * pop_myplace)) , 'ConfirmedCases_Pred1'] =(0.05 * pop_myplace)
 ax = tt.query('Place == @myplace' ).set_index('Date')[['ConfirmedCases','ConfirmedCases_Pred1']].plot(figsize=(15, 5), title=myplace, ax=axs[0])
 dat = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','Fatalities_Log']].dropna()
if len(dat)< 5:
tt.loc[(tt['Place'] == myplace), 'Fatalities_Pred1'] = tt.loc[(tt['Place'] == myplace)]['ConfirmedCases_Pred1'] * 0.0001
elif tt.query('Place == @myplace')['Fatalities'].max() < 5:
tt.loc[(tt['Place'] == myplace), 'Fatalities_Pred1'] = tt.loc[(tt['Place'] == myplace)]['ConfirmedCases_Pred1'] * 0.0001
else:
X = dat['Days_Since_Ten_Cases']
y = dat['Fatalities_Log']
y = y.cummax()
dat_all = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','Fatalities_Log']]
X_pred = dat_all['Days_Since_Ten_Cases']
en = ElasticNet()
en.fit(X.values.reshape(-1, 1), y.values)
 preds = en.predict(X_pred.values.reshape(-1, 1))
tt.loc[(tt['Place'] == myplace)&(tt['Days_Since_Ten_Cases'] >= 0), 'Fatalities_Log_Pred1'] = preds
tt.loc[(tt['Place'] == myplace), 'Fatalities_Pred1'] = tt['Fatalities_Log_Pred1'].apply(np.expm1)
 pop_myplace = tt.query('Place == @myplace')['Population_final'].values[0]
tt.loc[(tt['Place'] == myplace)&(tt['Fatalities_Pred1'] >(0.0001 * pop_myplace)) , 'Fatalities_Pred1'] =(0.0001 * pop_myplace)
 ax = tt.query('Place == @myplace' ).set_index('Date')[['Fatalities','Fatalities_Pred1']].plot(figsize=(15, 5), title=myplace, ax=axs[1])
 plt.show()
except:
print(f'============= FAILED FOR {myplace} =============')
<feature_engineering>
def create_feature_map(features):
outfile = open('xgb.fmap', 'w')
 for i, feat in enumerate(features):
outfile.write('{0}\t{1}\tq
'.format(i, feat))
outfile.close()
tt['ConfirmedCases_Pred1'] = tt[['ConfirmedCases','ConfirmedCases_Pred1']].max(axis=1)
 tt['Fatalities_Pred1'] = tt[['Fatalities','Fatalities_Pred1']].max(axis=1)
 tt['ConfirmedCases_Pred1'] = tt['ConfirmedCases_Pred1'].fillna(0)
 tt['Fatalities_Pred1'] = tt['Fatalities_Pred1'].fillna(0)
 tt.loc[~tt['ConfirmedCases'].isna() , 'ConfirmedCases_Pred1'] = tt.loc[~tt['ConfirmedCases'].isna() ]['ConfirmedCases']
tt.loc[~tt['Fatalities'].isna() , 'Fatalities_Pred1'] = tt.loc[~tt['Fatalities'].isna() ]['Fatalities']
tt['ConfirmedCases_Pred1'] = tt.groupby('Place')['ConfirmedCases_Pred1'].transform('cummax')
 tt['Fatalities_Pred1'] = tt.groupby('Place')['Fatalities_Pred1'].transform('cummax' )<sort_values>
if(trainpredictions is None):
trainpredictions = predictions
else:
trainpredictions += predictions
predictions = clf.predict(dtest, ntree_limit=limit)
 if(testpredictions is None):
testpredictions = predictions
else:
testpredictions += predictions
imp = get_importance(clf, features)
<prepare_x_and_y><EOS> 
y_pred =(testpredictions/folds > 0.4 ).astype(int)
 submission = pd.DataFrame({"Id": testdata.Id.values,
"Response": y_pred})
 submission[['Id', 'Response']].to_csv('xgbsubmission'+str(folds)+'.csv',
index=False)