Kaggle competition: COVID19 Global Forecasting (Week 2)
y = pd.get_dummies(labels)
 y.head()<split>
%matplotlib inline
for dirname, _, filenames in os.walk('/kaggle/input'):
print(dirname)
 data_path = Path('/kaggle/input/covid19-global-forecasting-week-1/')
 train = pd.read_csv(data_path / 'train.csv')
 test = pd.read_csv(data_path / 'test.csv')
 data_path = Path('/kaggle/input/covid19-global-forecasting-week-2/')
 train_2 = pd.read_csv(data_path / 'train.csv')
 test_2 = pd.read_csv(data_path / 'test.csv')
X_train, X_val, y_train, y_val = train_test_split(normalizeANDreshape(X, 0, 255), y, test_size = 0.20 )<train_model>
augment = ImageDataGenerator(rotation_range = 15,
width_shift_range = 0.35,
height_shift_range = 0.35,
zoom_range = 0.2, 
)
 augment.fit(X_train )<choose_model_class>
train['country+province'] = train['Country/Region'].fillna('')+ '-' + train['Province/State'].fillna('')
 train_2['country+province'] = train_2['Country_Region'].fillna('')+ '-' + train_2['Province_State'].fillna('')
 df = train.groupby('country+province')[['Lat', 'Long']].mean()
df.loc['United Kingdom-'] = df.loc['United Kingdom-United Kingdom']
df.loc['Diamond Princess-'] = df.loc['Cruise Ship-Diamond Princess']
df.loc['Denmark-'] = df.loc['Denmark-Denmark']
df.loc['France-'] = df.loc['France-France']
df.loc['Gambia-'] = df.loc['Gambia, The-']
df.loc['Netherlands-'] = df.loc['Netherlands-Netherlands']
df.loc['Dominica-'] =(15.3, -61.383333)
 df.loc['Angola-'] =(-8.830833, 13.245)
 df.loc['Bahamas-'] =(25.066667, -77.333333)
 df.loc['Belize-'] =(17.498611, -88.188611)
 df.loc['Cabo Verde-'] =(14.916667, -23.516667)
 df.loc['Chad-'] =(12.134722, 15.055833)
 df.loc['Denmark-Greenland'] =(64.181389, -51.694167)
 df.loc['El Salvador-'] =(13.698889, -89.191389)
 df.loc['Eritrea-'] =(15.322778, 38.925)
 df.loc['Fiji-'] =(-18.166667, 178.45)
 df.loc['France-Martinique'] =(14.666667, -61)
 df.loc['France-New Caledonia'] =(-22.2758, 166.458)
 df.loc['Grenada-'] =(12.05, -61.75)
 df.loc['Guinea-Bissau-'] =(11.85, -15.566667)
 df.loc['Haiti-'] =(18.533333, -72.333333)
 df.loc['Laos-'] =(17.966667, 102.6)
 df.loc['Libya-'] =(32.887222, 13.191389)
 df.loc['Madagascar-'] =(-18.933333, 47.516667)
 df.loc['Mali-'] =(12.639167, -8.002778)
 df.loc['Mozambique-'] =(-25.966667, 32.583333)
 df.loc['Netherlands-Sint Maarten'] =(18.052778, -63.0425)
 df.loc['Nicaragua-'] =(12.136389, -86.251389)
 df.loc['Niger-'] =(13.511667, 2.125278)
 df.loc['Papua New Guinea-'] =(-9.478889, 147.149444)
 df.loc['Saint Kitts and Nevis-'] =(17.3, -62.733333)
 df.loc['Syria-'] =(33.513056, 36.291944)
 df.loc['Timor-Leste-'] =(-8.566667, 125.566667)
 df.loc['Uganda-'] =(0.313611, 32.581111)
 df.loc['Zimbabwe-'] =(-17.829167, 31.052222)
 df.loc['United Kingdom-Bermuda'] =(32.293, -64.782)
 df.loc['United Kingdom-Isle of Man'] =(54.145, -4.482)
 train_2['Lat'] = train_2['country+province'].apply(lambda x: df.loc[x, 'Lat'])
 train_2['Long'] = train_2['country+province'].apply(lambda x: df.loc[x, 'Long'])
 train_2.head()
def makeCNN(shape):
model = Sequential()
model.add(Conv2D(filters = 64, kernel_size =(4,4), padding = 'same', activation = 'relu', input_shape = shape))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(BatchNormalization())
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3), padding = 'same', activation = 'relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Conv2D(filters = 64, kernel_size =(3,3), padding = 'same', activation = 'relu'))
model.add(Conv2D(filters = 128, kernel_size =(2,2), padding = 'same', activation = 'relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Flatten())
 model.add(Dense(64, activation = 'relu'))
model.add(Dropout(0.25))
model.add(Dense(10, activation = 'softmax'))
return model
model = makeCNN(( 28, 28, 1))
model.summary()<choose_model_class>
train_2.to_csv("train2_latlong.csv",index=False )
model.compile(optimizer = Adam() ,
loss = 'categorical_crossentropy',
metrics = 'accuracy' )<train_model>
test_2['country+province'] = test_2['Country_Region'].fillna('')+ '-' + test_2['Province_State'].fillna('')
 test_2['Lat'] = test_2['country+province'].apply(lambda x: df.loc[x, 'Lat'])
 test_2['Long'] = test_2['country+province'].apply(lambda x: df.loc[x, 'Long'])
 test_2.head()
epochs = 50
batch_size = 64
history = model.fit_generator(augment.flow(X_train,y_train, batch_size=batch_size),
epochs = epochs,
steps_per_epoch=len(X_train)// batch_size,
validation_data =(X_val,y_val),
verbose = 1,
use_multiprocessing = True,
workers = 2
)<predict_on_test>
test_2.to_csv("test2_latlong.csv",index=False )
validation_predictions = model.predict_classes(X_val)
 confusion = confusion_matrix(validation_predictions,y_val.idxmax(axis=1))
print(confusion )<predict_on_test>
test = normalizeANDreshape(test, 0, 255)
 predictions = model.predict_classes(test)
 print(predictions[0:5] )<save_to_csv>
import lightgbm as lgb
from fastai.tabular import *
from sklearn import preprocessing
import datetime
Id = []
for i in range(len(test)) :
Id.append(i+1)
 output = pd.DataFrame({'ImageID': Id, 'Label': predictions})
 output.to_csv('predictions.csv', index=False )<set_options>
train = pd.read_csv("train2_latlong.csv")
 test = pd.read_csv("test2_latlong.csv")
 sub = pd.read_csv(".. /input/covid19-global-forecasting-week-2/submission.csv" )
%matplotlib inline 
<set_options>
train = train.append(test[test['Date']>'2020-03-26'] )
%matplotlib inline 
<load_from_csv>
def rmsle(y_true, y_pred):
return np.sqrt(np.mean(np.power(np.log1p(y_pred)- np.log1p(y_true), 2)))
 def mape(y_true, y_pred):
return np.mean(np.abs(y_pred -y_true)*100/(y_true+1))
datadir = '.. /input'
gatrain = pd.read_csv(os.path.join(datadir,'gender_age_train.csv'),
index_col='device_id')
 gatest = pd.read_csv(os.path.join(datadir,'gender_age_test.csv'),
index_col = 'device_id')
 phone = pd.read_csv(os.path.join(datadir,'phone_brand_device_model.csv'))
phone = phone.drop_duplicates('device_id',keep='first' ).set_index('device_id')
 events = pd.read_csv(os.path.join(datadir,'events.csv'),
parse_dates=['timestamp'], index_col='event_id')
 appevents = pd.read_csv(os.path.join(datadir,'app_events.csv'),
usecols=['event_id','app_id','is_active'],
dtype={'is_active':bool})
 applabels = pd.read_csv(os.path.join(datadir,'app_labels.csv'))<load_from_csv>
train['Date'] = pd.to_datetime(train['Date'], format='%Y-%m-%d' )
datadir = '.. /input'
gatrain = pd.read_csv(os.path.join(datadir,'gender_age_train.csv'),
index_col='device_id')
 gatest = pd.read_csv(os.path.join(datadir,'gender_age_test.csv'),
index_col = 'device_id')
 phone = pd.read_csv(os.path.join(datadir,'phone_brand_device_model.csv'))
phone = phone.drop_duplicates('device_id',keep='first' ).set_index('device_id')
 events = pd.read_csv(os.path.join(datadir,'events.csv'),
parse_dates=['timestamp'], index_col='event_id')
 appevents = pd.read_csv(os.path.join(datadir,'app_events.csv'),
usecols=['event_id','app_id','is_active'],
dtype={'is_active':bool})
 applabels = pd.read_csv(os.path.join(datadir,'app_labels.csv'))<feature_engineering>
train['day_dist'] = train['Date']-train['Date'].min()
train['day_dist'] = train['day_dist'].dt.days
cat_cols = train.dtypes[train.dtypes=='object'].keys()
cat_cols
gatrain['trainrow'] = np.arange(gatrain.shape[0])
 gatest['testrow'] = np.arange(gatest.shape[0] )<categorify>
for cat_col in cat_cols:
train[cat_col].fillna('no_value', inplace = True)
 train['place'] = train['Province_State']+'_'+train['Country_Region']
for cat_col in ['place']:
le = preprocessing.LabelEncoder()
le.fit(train[cat_col])
 train[cat_col]=le.transform(train[cat_col] )
brandencoder = LabelEncoder().fit(phone.phone_brand)
 phone['brand'] = brandencoder.transform(phone['phone_brand'])
 gatrain['brand'] = phone['brand']
gatest['brand'] = phone['brand']
Xtr_brand = csr_matrix(( np.ones(gatrain.shape[0]), 
(gatrain.trainrow, gatrain.brand)))
 Xte_brand = csr_matrix(( np.ones(gatest.shape[0]), 
(gatest.testrow, gatest.brand)))
 print('Brand features: train shape {}, test shape {}'.format(Xtr_brand.shape, Xte_brand.shape))<categorify>
brandencoder = LabelEncoder().fit(phone.phone_brand)
 phone['brand'] = brandencoder.transform(phone['phone_brand'])
 gatrain['brand'] = phone['brand']
gatest['brand'] = phone['brand']
Xtr_brand = csr_matrix(( np.ones(gatrain.shape[0]), 
(gatrain.trainrow, gatrain.brand)))
 Xte_brand = csr_matrix(( np.ones(gatest.shape[0]), 
(gatest.testrow, gatest.brand)))
 print('Brand features: train shape {}, test shape {}'.format(Xtr_brand.shape, Xte_brand.shape))<categorify>
val = train[(train['Date']>='2020-03-19')&(train['Id'].isnull() ==False)]
m = phone.phone_brand.str.cat(phone.device_model)
 modelencoder = LabelEncoder().fit(m)
 phone['model'] = modelencoder.transform(m)
 gatrain['model'] = phone['model']
gatest['model'] = phone['model']
Xtr_model = csr_matrix(( np.ones(gatrain.shape[0]), 
(gatrain.trainrow, gatrain.model)))
 Xte_model = csr_matrix(( np.ones(gatest.shape[0]), 
(gatest.testrow, gatest.model)))
 print('Model features: train shape {}, test shape {}'.format(Xtr_model.shape, Xte_model.shape))<create_dataframe>
y_ft = train["Fatalities"]
y_val_ft = val["Fatalities"]
y_cc = train["ConfirmedCases"]
y_val_cc = val["ConfirmedCases"]
appencoder = LabelEncoder().fit(appevents.app_id)
 appevents['app'] = appencoder.transform(appevents.app_id)
 napps = len(appencoder.classes_)
 deviceapps =(appevents.merge(events[['device_id']], how='left',left_on='event_id',right_index=True)
.groupby(['device_id','app'])['app'].agg(['size'])
.merge(gatrain[['trainrow']], how='left', left_index=True, right_index=True)
.merge(gatest[['testrow']], how='left', left_index=True, right_index=True)
.reset_index())
 deviceapps.head()<prepare_x_and_y>
params = {
"objective": "regression",
"boosting": 'gbdt',
"num_leaves": 1280,
"learning_rate": 0.05,
"feature_fraction": 0.9,
"reg_lambda": 2,
"metric": "rmse",
'min_data_in_leaf':20
}
d = deviceapps.dropna(subset=['trainrow'])
 Xtr_app = csr_matrix(( np.ones(d.shape[0]),(d.trainrow, d.app)) ,
shape=(gatrain.shape[0],napps))
d = deviceapps.dropna(subset=['testrow'])
 Xte_app = csr_matrix(( np.ones(d.shape[0]),(d.testrow, d.app)) ,
shape=(gatest.shape[0],napps))
print('Apps data: train shape {}, test shape {}'.format(Xtr_app.shape, Xte_app.shape))<categorify>
dates = dates[dates>'2020-03-26']
len(dates )
applabels = applabels.loc[applabels.app_id.isin(appevents.app_id.unique())]
applabels['app'] = appencoder.transform(applabels.app_id)
 labelencoder = LabelEncoder().fit(applabels.label_id)
 applabels['label'] = labelencoder.transform(applabels.label_id)
 nlabels = len(labelencoder.classes_ )<categorify>
drop_cols = ['Id', 'ConfirmedCases', 'Fatalities','day_dist', 'Province_State', 'Country_Region', 'Date', 'country+province']
applabels = applabels.loc[applabels.app_id.isin(appevents.app_id.unique())]
applabels['app'] = appencoder.transform(applabels.app_id)
 labelencoder = LabelEncoder().fit(applabels.label_id)
 applabels['label'] = labelencoder.transform(applabels.label_id)
 nlabels = len(labelencoder.classes_ )<merge>
places = train["country+province"].unique()
len(places )
devicelabels =(deviceapps[['device_id','app']] 
.merge(applabels[['app','label']])
.groupby(['device_id','label'])['app'].agg(['size'])
.merge(gatrain[['trainrow']], how='left', left_index=True, right_index=True)
.merge(gatest[['testrow']], how='left', left_index=True, right_index=True)
.reset_index())
 devicelabels.head()<correct_missing_values>
d = devicelabels.dropna(subset=['trainrow'])
 Xtr_label = csr_matrix(( np.ones(d.shape[0]),(d.trainrow, d.label)) ,
shape=(gatrain.shape[0],nlabels))
d = devicelabels.dropna(subset=['testrow'])
 Xte_label = csr_matrix(( np.ones(d.shape[0]),(d.testrow, d.label)) ,
shape=(gatest.shape[0],nlabels))
print('Labels data: train shape {}, test shape {}'.format(Xtr_label.shape, Xte_label.shape))<concatenate>
places_dict={}
for place in places:
initdate = min(train.loc[(train["country+province"]==place)&(train["ConfirmedCases"]>0)]["Date"])
 places_dict[place] = initdate.date()
places_dict
Xtrain = hstack(( Xtr_brand, Xtr_model, Xtr_app, Xtr_label), format='csr')
 Xtest = hstack(( Xte_brand, Xte_model, Xte_app, Xte_label), format='csr')
 print('All features: train shape {}, test shape {}'.format(Xtrain.shape, Xtest.shape))<concatenate>
days_from_first_case = []
for i in range(len(train)) :
row = train.iloc[i]
row = row.to_dict()
initdate = places_dict[row["country+province"]]
currdate = row["Date"].date()
days =(currdate - initdate ).days
days_from_first_case.append(max(0,days))
Xtrain = hstack(( Xtr_brand, Xtr_model, Xtr_app, Xtr_label), format='csr')
 Xtest = hstack(( Xte_brand, Xte_model, Xte_app, Xte_label), format='csr')
 print('All features: train shape {}, test shape {}'.format(Xtrain.shape, Xtest.shape))<categorify>
train['Days_from_first_case'] = days_from_first_case
targetencoder = LabelEncoder().fit(gatrain.group)
 y = targetencoder.transform(gatrain.group)
 nclasses = len(targetencoder.classes_ )<compute_train_metric>
train.to_csv("finaltrain.csv",index=False )
def score(clf, random_state = 0):
kf = StratifiedKFold(y, n_folds=5, shuffle=True, random_state=random_state)
 pred = np.zeros(( y.shape[0],nclasses))
for itrain, itest in kf:
Xtr, Xte = Xtrain[itrain, :], Xtrain[itest, :]
ytr, yte = y[itrain], y[itest]
clf.fit(Xtr, ytr)
 pred[itest,:] = clf.predict_proba(Xte)
 return log_loss(yte, pred[itest, :])
 print("{:.5f}".format(log_loss(yte, pred[itest,:])) , end=' ')
 print('')
 return log_loss(y, pred )<compute_test_metric>
test[test['Country_Region']=='India']
score(LogisticRegression(C=0.02))<choose_model_class>
train_sub = pd.read_csv(".. /input/covid19-global-forecasting-week-2/train.csv" )
score(LogisticRegression(C=0.02, multi_class='multinomial',solver='lbfgs'))<compute_test_metric>
test = pd.merge(test,train_sub[['Province_State','Country_Region','Date','ConfirmedCases','Fatalities']], on=['Province_State','Country_Region','Date'], how='left' )
clf = LogisticRegression(C=0.02, multi_class='multinomial',solver='lbfgs')
 clf.fit(Xtrain, y)
 pred = pd.DataFrame(clf.predict_proba(Xtest), index = gatest.index, columns=targetencoder.classes_)
 pred.head()<save_to_csv>
test.loc[test['ConfirmedCases_x'].isnull() ==True, 'ConfirmedCases_x'] = test.loc[test['ConfirmedCases_x'].isnull() ==True, 'ConfirmedCases_y']
pred.to_csv('logreg_subm.csv',index=True )<set_options>
test.loc[test['Fatalities_x'].isnull() ==True, 'Fatalities_x'] = test.loc[test['Fatalities_x'].isnull() ==True, 'Fatalities_y']
%matplotlib inline<define_variables>
last_amount = test.loc[(test['Country_Region']=='Italy')&(test['Date']=='2020-03-26'),'ConfirmedCases_x']
def sample_horse(size=1):
return np.maximum(0, np.random.normal(5,2,size))
def sample_ball(size=1):
return np.maximum(0, 1 + np.random.normal(1,0.3,size))
def sample_bike(size=1):
return np.maximum(0, np.random.normal(20,10,size))
def sample_train(size=1):
return np.maximum(0, np.random.normal(10,5,size))
def sample_coal(size=1):
return 47 * np.random.beta(0.5,0.5,size)
 def sample_book(size=1):
return np.random.chisquare(2,size)
 def sample_doll(size=1):
return np.random.gamma(5,1,size)
 def sample_block(size=1):
return np.random.triangular(5,10,20,size)
 def sample_gloves(size=1):
dist1 = 3.0 + np.random.rand(size)
 dist2 = np.random.rand(size)
 toggle = np.random.rand(size)< 0.3
dist2[toggle] = dist1[toggle]
return dist2
samplers = {
"horse": sample_horse,
"ball": sample_ball,
"bike": sample_bike,
"train": sample_train,
"coal": sample_coal,
"book": sample_book,
"doll": sample_doll,
"blocks": sample_block,
"gloves": sample_gloves
}
def sample(gift, quantity=1, size=1):
return np.sum(samplers[gift](quantity * size ).reshape(quantity, size), axis=0)
 print(sample("horse", 2, 10))<statistical_test>
last_fat = test.loc[(test['Country_Region']=='Italy')&(test['Date']=='2020-03-24'),'Fatalities_x']
def bag_name(bag):
return str(list(map(lambda gift: "{}({})".format(gift, bag[gift]), sorted(bag.keys()))))
def create_bag_weight_sampler(bag):
def bag_weight_sampler(size=1):
weight = np.array([0.0]*size)
 for gift in sorted(bag.keys()):
weight += sample(gift, bag[gift], size)
 return weight
return bag_weight_sampler, bag_name(bag)
 bag = { "horse": 1, "ball": 2 }
bag_weight_sampler, name = create_bag_weight_sampler(bag)
 print("Sampling from bag {}: {}".format(name, bag_weight_sampler(3)) )<statistical_test>
i, k = 0, 35
for date in dates:
k = k-1
i = i + 1
test.loc[(test['Country_Region']=='Italy')&(test['Date']==date),'ConfirmedCases_x'] = last_amount.values[0]+i*(5000-(100*i))
test.loc[(test['Country_Region']=='Italy')&(test['Date']==date),'Fatalities_x'] = last_fat.values[0]+i*(800-(10*i))
np.random.seed(42)
 def get_gift_weight_distributions(gifts, size=10000):
def get_gift_weight_dsitribution(gift):
sampler = samplers[gift]
sample = sampler(size)
 return np.mean(sample), np.var(sample)
 distributions = np.zeros(( len(gifts), 2))
for i, gift in enumerate(gifts):
distributions[i, :] = get_gift_weight_dsitribution(gift)
 return distributions
gifts = sorted(samplers.keys())
 print("Canonical gift order: {}
".format(gifts))
gift_weight_distributions = get_gift_weight_distributions(gifts)
 print(pd.DataFrame(data=gift_weight_distributions, index=gifts, columns=["mean", "std"]))<categorify>
test.loc[(test['Country_Region']=='Italy')]
def create_candidate_bags(max_quantities):
gift_counts = []
for max_quantity in max_quantities:
gift_counts.append(np.arange(max_quantity))
return cartesian(gift_counts)
 mixed_item_candiadte_bags = create_candidate_bags(mixed_item_max_quantities)
 print("Created candiadate bags: {}".format(mixed_item_candiadte_bags.shape))<groupby>
sub = test[['ForecastId', 'ConfirmedCases_x','Fatalities_x']]
sub.columns = ['ForecastId', 'ConfirmedCases', 'Fatalities']
def get_bag_weight_distributions(candidate_bags, min_mean=30, max_mean=50):
return np.dot(candidate_bags, gift_weight_distributions)
 def filter_by_mean(bags, distributions, min_mean=30, max_mean=50):
min_mask = mean_of(distributions)> min_mean
distributions = distributions[min_mask]
bags = bags[min_mask]
max_mask = mean_of(distributions)< max_mean
distributions = distributions[max_mask]
bags = bags[max_mask]
return bags, distributions
def mean_of(distributions):
return distributions[:,0]
mixed_item_bag_weight_distributions = get_bag_weight_distributions(mixed_item_candiadte_bags)
 mixed_item_candiadte_bags, mixed_item_bag_weight_distributions = \
filter_by_mean(mixed_item_candiadte_bags, mixed_item_bag_weight_distributions)
 print("Candidate bags left: {}".format(mixed_item_candiadte_bags.shape))<categorify>
sub.loc[sub['ConfirmedCases']<0, 'ConfirmedCases'] = 0
def get_low_weight_item_candidate_bags() :
bags = []
distributions = []
for gift in ["ball", "book", "gloves"]:
max_quantities = get_low_weight_item_bags_max_quantities_for(gift)
 candiadte_bags = create_candidate_bags(max_quantities)
 bags.append(candiadte_bags)
 cadidate_bag_weight_distributions = get_bag_weight_distributions(candiadte_bags)
 distributions.append(cadidate_bag_weight_distributions)
 return np.vstack(bags), np.vstack(distributions)
 def get_low_weight_item_bags_max_quantities_for(gift):
max_quantities = np.ceil(50 / gift_weight_distributions[:, 0])
 gift_index = gifts.index(gift)
 for i in range(len(max_quantities)) :
if not i == gift_index:
max_quantities[i] = 5
print("Gift {}: number of different bags: {}".format(gift, np.prod(max_quantities)))
 return max_quantities
low_weight_item_candidate_bags, low_weight_item_bag_weight_distributions = \
filter_by_mean(*get_low_weight_item_candidate_bags())
 print("Total number of canidate bags: {}".format(low_weight_item_candidate_bags.shape))<count_duplicates>
sub.loc[sub['Fatalities']<0, 'Fatalities'] = 0
def drop_duplicate(candidate_bags, distributions):
df = pd.DataFrame(data=np.hstack(( candidate_bags, distributions)) , columns=gifts + ["mean", "std"])
 df.drop_duplicates(subset=gifts, inplace=True)
 return df[gifts].values, df[["mean", "std"]].values
candidate_bags = np.vstack([mixed_item_candiadte_bags, low_weight_item_candidate_bags])
 bag_weight_distributions = np.vstack([mixed_item_bag_weight_distributions, low_weight_item_bag_weight_distributions])
 print("Combined candiadte bags: {}".format(candidate_bags.shape))
candidate_bags, bag_weight_distributions = drop_duplicate(candidate_bags, bag_weight_distributions)
 print("Final candidate bags without duplicates: {}".format(candidate_bags.shape))<statistical_test>
sub.to_csv('submission.csv',index=False )