Kaggle competition: COVID19 Global Forecasting (Week 4)
submission = pd.DataFrame(dict(
 image_name = image_names,
target = probs[:]))
submission = submission.sort_values('image_name')
 submission.to_csv('submission_model.csv', index=False )<import_modules>
warnings.filterwarnings('ignore')
!pip install -q efficientnet 
<set_options>
test = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 train = pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 df_pop = pd.read_csv('.. /input/population-by-country-2020/population_by_country_2020.csv')
 covid_pop = pd.read_csv('.. /input/covid19-global-forecasting-locations-population/locations_population.csv' )
def get_strategy() :
gpu = ""
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
gpu = tf.config.list_physical_devices("GPU")
 if len(gpu)== 1:
print('Running on GPU ', gpu)
 if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 tf.config.experimental_connect_to_cluster(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 GCS_PATH = KaggleDatasets().get_gcs_path('siim-isic-melanoma-classification')
 elif len(gpu)== 1:
strategy = tf.distribute.OneDeviceStrategy(device="/gpu:0")
 tf.config.optimizer.set_experimental_options({"auto_mixed_precision":True})
 GCS_PATH = "/kaggle/input/siim-isic-melanoma-classification/"
else:
strategy = tf.distribute.get_strategy()
GCS_PATH = "/kaggle/input/siim-isic-melanoma-classification/"
print("REPLICAS: ", strategy.num_replicas_in_sync)
 base_dir = "/kaggle/input/siim-isic-melanoma-classification/"
return strategy, GCS_PATH, base_dir
strategy,GCS_PATH, base_dir = get_strategy()<define_variables>
print("Max Date: ", max(train['Date']), ", Min Date: ", min(train['Date']))
print("Max Id: ", max(train['Id']), ", Unique Ids: ", train['Id'].nunique())
 print("Max Date: ", max(test['Date']), ", Min Date: ", min(test['Date']))
VERSION = 24
KEY_CHANGES = "Code cleanup, "
AUTO = tf.data.experimental.AUTOTUNE
ROT_ = 45.0
SHR_ = 2.0
HZOOM_ = 10.0
WZOOM_ = 10.0
HSHIFT_ = 10.0
WSHIFT_ = 10.0
DIM=384
TFREC_DIM=384
SEED = 42
TFREC_GCS_DIR = KaggleDatasets().get_gcs_path('melanoma-%ix%i'%(TFREC_DIM,TFREC_DIM))
PRV_TFREC_GCS_DIR = KaggleDatasets().get_gcs_path('isic2019-%ix%i'%(TFREC_DIM,TFREC_DIM))
BLN_TRAIN_MODEL=False
REPLICAS = strategy.num_replicas_in_sync
BATCH_SIZE = 16 * REPLICAS
EPOCHS=12
KFOLD_SPLITS=5
TTA=25<save_to_csv>
train.isnull().sum()
log_sno = 1
log_start = datetime.datetime.now()
def write_log(data_type, data):
global log_sno
filename = "log_melanoma_classification.txt"
prv_filename = ".. /input/amazingly-fast-kernel/" + filename
if(not os.path.exists(filename)) and os.path.exists(prv_filename):
os.popen('cp ' + prv_filename + ' ' + filename)
 f = open(filename,"a")
 log_cur = datetime.datetime.now()
delta = log_cur - log_start
f.write("{},{},{},{},{},{}".format(VERSION, log_sno, log_start, data_type, data, delta.total_seconds()))
log_sno = log_sno + 1
f.close()
write_log("KEY_CHANGES", KEY_CHANGES )<load_from_csv>
train.isnull().sum()
train_data = pd.read_csv(base_dir + "train.csv")
 print(display(HTML(train_data.head(1 ).to_html())))
 print(train_data["target"].value_counts() )<feature_engineering>
cond1 = train.Fatalities >=1
cond2 = train.ConfirmedCases >= 1
train_Fatal = train[['Country_Region','Date']][(cond1)]
train_Confirm = train[['Country_Region','Date']][(cond2)]
Confirm_Min_Max = train_Confirm.groupby("Country_Region", as_index=False)["Date"].agg(["min","max"])
 Fatal_Min_Max = train_Fatal.groupby("Country_Region", as_index=False)["Date"].agg(["min","max"])
 Country_Dates = pd.merge(Confirm_Min_Max, Fatal_Min_Max, how='left', on=['Country_Region'])
 Country_Dates = Country_Dates.rename(columns={'min_x': 'fst_Confirmed', 'max_x': 'last_Confirmed', 'min_y': 'fst_Fatal','max_y': 'last_Fatal'})
 Country_Dates['Fatality_Days'] = pd.to_datetime(Country_Dates['last_Fatal'])- pd.to_datetime(Country_Dates['fst_Fatal'])
 Country_Dates = Country_Dates.reset_index()
train_data["anatom_site_general_challenge"].fillna("Unknown", inplace=True)
 group_data = train_data.groupby(["anatom_site_general_challenge"])["benign_malignant"].value_counts().unstack(-1)
 group_data["perc_malignant"] = np.round(( group_data["malignant"] * 100)/(group_data["benign"] + group_data["malignant"]),2)
 group_data.sort_values("perc_malignant", inplace=True )<normalization>
Country_Dates['Fatality_Days'] = Country_Dates['Fatality_Days'] / np.timedelta64(1, 'D')
 Country_Dates.head(5 )
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def aug(image, DIM=DIM):
XDIM = DIM%2
rot = ROT_ * tf.random.normal([1], dtype='float32')
 shr = SHR_ * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ HZOOM_
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ WZOOM_
h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32')
 w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
img = tf.reshape(d,[DIM, DIM,3])
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 return img
def aug_img_label(img, label):
img = aug(img)
 return img, label
def aug_img(img):
img = aug(img)
 return img<split>
cond1 = train.Date == '2020-04-12'
Country_Counts = train[['Country_Region','Date','Fatalities','ConfirmedCases']][(cond1)]
df_Country2 = pd.merge(Country_Dates, Country_Counts, how='left', on=['Country_Region'])
 df_Country2['Fatality_Rate'] = round(df_Country2['Fatalities']/pd.to_numeric(df_Country2['Fatality_Days']),2)
 df_Country2['Fatality_Rate'] = round(df_Country2['Fatalities']/pd.to_numeric(df_Country2['Fatality_Days']),2)
 df_Country2['Fatality_Rate'] = df_Country2['Fatality_Rate'].fillna(0)
 df_Country2 = df_Country2.replace([np.inf, -np.inf], np.nan)
 df_Country3 = df_Country2[df_Country2['Fatality_Rate'] > 0].groupby(['Country_Region'])[["Fatality_Rate"]].sum().nlargest(8,'Fatality_Rate')
 df_Country3 = df_Country3.reset_index()
def parse_rec_train(data):
feature_set = {
'image': tf.io.FixedLenFeature([], tf.string),
'target': tf.io.FixedLenFeature([], tf.int64)
 }
features = tf.io.parse_single_example(data, features= feature_set)
 return features
def parse_rec_validate(data):
feature_set = {
'image': tf.io.FixedLenFeature([], tf.string),
'target': tf.io.FixedLenFeature([], tf.int64),
'image_name': tf.io.FixedLenFeature([], tf.string)
 }
features = tf.io.parse_single_example(data, features= feature_set)
 return features
def parse_rec_test(data):
feature_set = {
'image': tf.io.FixedLenFeature([], tf.string),
'image_name': tf.io.FixedLenFeature([], tf.string)
 }
features = tf.io.parse_single_example(data, features= feature_set)
 return features
def process_img(img):
img = tf.image.decode_image(img)
 img = tf.ensure_shape(img,(TFREC_DIM,TFREC_DIM,3))
img = tf.image.resize(img, [DIM,DIM])
 img = float(img)/255.00
return tf.cast(img, tf.float32)
 def get_img_label(features):
target = features["target"]
features.pop("target")
 img = process_img(features["image"])
 return img, target
def get_img(features, label=None):
img = process_img(features["image"])
 return img
def get_img_and_name(features, label=None):
img = process_img(features["image"])
 image_name = features["image_name"]
return img, image_name
def get_img_name(features):
image_name = features["image_name"]
return image_name<define_variables>
confirmed_China = train[train['Country_Region']=='China'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_China = train[train['Country_Region']=='China'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_China = confirmed_China.join(fatalities_China)
 confirmed_Italy = train[train['Country_Region']=='Italy'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_Italy = train[train['Country_Region']=='Italy'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_Italy = confirmed_Italy.join(fatalities_Italy)
 confirmed_Spain = train[train['Country_Region']=='Spain'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_Spain = train[train['Country_Region']=='Spain'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_Spain = confirmed_Spain.join(fatalities_Spain)
 confirmed_Iran = train[train['Country_Region']=='Iran'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_Iran = train[train['Country_Region']=='Iran'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_Iran = confirmed_Iran.join(fatalities_Iran)
 confirmed_France = train[train['Country_Region']=='France'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_France = train[train['Country_Region']=='France'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_France = confirmed_France.join(fatalities_France)
 confirmed_Netherlands = train[train['Country_Region']=='Netherlands'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_Netherlands = train[train['Country_Region']=='Netherlands'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_Netherlands = confirmed_Netherlands.join(fatalities_Netherlands)
 confirmed_UK = train[train['Country_Region']=='United Kingdom'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_UK = train[train['Country_Region']=='United Kingdom'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_UK = confirmed_UK.join(fatalities_UK)
 confirmed_USA = train[train['Country_Region']=='US'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_USA = train[train['Country_Region']=='US'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_USA = confirmed_USA.join(fatalities_USA)
tfrec_files_train_all = np.sort(np.array(tf.io.gfile.glob(TFREC_GCS_DIR + '/train*.tfrec')))
 tfrec_files_train_2019 = np.sort(np.array(tf.io.gfile.glob(PRV_TFREC_GCS_DIR + '/train%.2i*.tfrec'%x for x in range(1, 31, 2))))
tfrec_files_train_2018 = np.sort(np.array(tf.io.gfile.glob(PRV_TFREC_GCS_DIR + '/train%.2i*.tfrec'%x for x in range(0, 30, 2))))
tfrec_files_test = np.sort(np.array(tf.io.gfile.glob(TFREC_GCS_DIR + '/test*.tfrec')))
 dataset_peek = tf.data.TFRecordDataset(tfrec_files_train_all)
 for data in dataset_peek.take(1):
example = tf.train.Example()
example.ParseFromString(data.numpy())
 print(str(example)[-600:] )<choose_model_class>
df_pop_sel = df_pop[['Country(or dependency)','Population(2020)','Density(P/Km²)','Land Area(Km²)','Med.Age','Urban Pop %']]
df_pop_sel.columns = ['Country_Region', 'Population(2020)', 'Density(P/Km²)','Land Area(Km²)', 'Med.Age', 'Urban Pop %']
df_pop_sel['Urban Pop %'] = df_pop_sel['Urban Pop %'].str.rstrip('%')
 df_pop_sel.loc[df_pop_sel['Country_Region']=='United States', 'Country_Region'] = 'US'
df_country_pop = pd.merge(df_pop_sel, df_Country, how='inner', on=['Country_Region'])
 df_country_pop
EFNS = [efn.EfficientNetB0, efn.EfficientNetB4, efn.EfficientNetB6, efn.EfficientNetB6, efn.EfficientNetB6]
def build_model(fold):
if BLN_TRAIN_MODEL:
if 1==2:
base_model = efn.EfficientNetB0(weights='imagenet', include_top=False, input_shape=(DIM,DIM,3))
x = base_model.output
x = GlobalAveragePooling2D()(x)
 x = Dense(1024 )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation('relu' )(x)
 x = tf.keras.layers.Dropout(0.02 )(x)
 predictions = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=base_model.input, outputs=predictions)
 inp = tf.keras.layers.Input(shape=(DIM,DIM,3))
base = EFNS[fold](input_shape=(DIM,DIM,3),weights='imagenet',include_top=False)
 x = base(inp)
 x = tf.keras.layers.GlobalAveragePooling2D()(x)
 x = tf.keras.layers.Dense(1,activation='sigmoid' )(x)
 model = tf.keras.Model(inputs=inp,outputs=x)
 opt = tf.keras.optimizers.Adam(learning_rate=0.001)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05)
 model.compile(optimizer=opt,loss=loss,metrics=['AUC'])
 return model
else:
model_path = "/kaggle/input/model-data-pipeline-v18/model.h5"
model = tf.keras.models.load_model(model_path)
 opt = tf.keras.optimizers.Adam(lr=0.001)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05, name='binary_crossentropy')
 model.compile(optimizer=opt, loss=loss, metrics=['accuracy',tf.keras.metrics.AUC() ])
 return model 
<init_hyperparams>
train_fil['Date'] = pd.to_datetime(train_fil['Date'])
 train_fil['Day_num'] = preprocessing.LabelEncoder().fit_transform(train_fil.Date)
 train_fil['Day'] = train_fil['Date'].dt.day
train_fil['Month'] = train_fil['Date'].dt.month
train_fil['Year'] = train_fil['Date'].dt.year
train_fil_copy = train_fil
train_fil['Province_State'].fillna("None", inplace=True)
 train_fil[train_fil.iloc[:,0:].isnull().any(axis = 1)].iloc[:,0:].head()
def get_lr_callback(batch_size):
lr_start = 0.000005
lr_max = 0.000020 * strategy.num_replicas_in_sync
lr_min = 0.000001
lr_ramp_ep = 5
lr_sus_ep = 0
lr_decay = 0.8
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback
lr = get_lr_callback(BATCH_SIZE)
 es = tf.keras.callbacks.EarlyStopping(monitor='val_auc', mode='max', verbose=1, patience=3)
<categorify>
df_train_pop = train_fil.merge(covid_pop_sel, left_on = ['Country_Region','Province_State'], right_on = ['Country.Region','Province.State'], how = 'left')
dataset_test = tf.data.TFRecordDataset(tfrec_files_test ).map(parse_rec_test, num_parallel_calls=AUTO)
 img_name_list_test = list(dataset_test.map(get_img_name, num_parallel_calls=AUTO ).as_numpy_iterator())
 dataset_test_raw = dataset_test.map(get_img, num_parallel_calls=AUTO )<groupby>
display(covid_pop_sel[covid_pop_sel['Country.Region']=='Italy'])
 display(train_fil[train_fil['Country_Region']=='Italy'] )
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )<string_transform>
arr_test_pred = []
arr_test_pred_min = []
arr_test_pred_max = []
if BLN_TRAIN_MODEL:
kf = KFold(n_splits=KFOLD_SPLITS, shuffle=True,random_state=SEED)
 for fold ,(train_idx, val_idx)in enumerate(kf.split(range(len(tfrec_files_train_all)))) :
print(train_idx, val_idx)
 tfrec_files_train = np.array(tfrec_files_train_all)[list(train_idx)]
if fold >= 3:
additional_files = np.array(tfrec_files_train_2019)[list(train_idx)]
else:
additional_files = np.array(tfrec_files_train_2018)[list(train_idx)]
tfrec_files_train = np.concatenate([tfrec_files_train, additional_files])
 np.random.shuffle(tfrec_files_train)
 tfrec_files_valid = np.array(tfrec_files_train_all)[list(val_idx)]
print("num_steps:", count_data_items(tfrec_files_train)/BATCH_SIZE)
 temp_dataset = tf.data.TFRecordDataset(tfrec_files_train ).shuffle(1024*8 ).map(parse_rec_train, num_parallel_calls=AUTO ).map(get_img_label, num_parallel_calls=AUTO)
 temp_dataset = temp_dataset.map(aug_img_label, num_parallel_calls=AUTO)
 dataset_train = temp_dataset.batch(BATCH_SIZE, drop_remainder=True ).prefetch(tf.data.experimental.AUTOTUNE)
 dataset_valid = tf.data.TFRecordDataset(tfrec_files_valid ).map(parse_rec_train, num_parallel_calls=AUTO ).map(get_img_label, num_parallel_calls=AUTO ).batch(BATCH_SIZE, drop_remainder=True ).prefetch(tf.data.experimental.AUTOTUNE)
 K.clear_session()
with strategy.scope() :
model = build_model(fold)
 sv = tf.keras.callbacks.ModelCheckpoint('fold-%i.h5'%fold, monitor='val_auc', verbose=0, save_best_only=True,
save_weights_only=True, mode='max', save_freq='epoch')
 model.fit(dataset_train, epochs=EPOCHS, verbose=1, callbacks=[sv, lr, es], validation_data = dataset_valid)
 model.load_weights('fold-%i.h5'%fold)
 arr_pred = []
for i in range(TTA):
dataset_test = dataset_test_raw.map(aug_img, num_parallel_calls=AUTO ).batch(BATCH_SIZE ).prefetch(tf.data.experimental.AUTOTUNE)
 arr_pred.append(model.predict(dataset_test, verbose=1))
arr_test_pred.append(np.stack(arr_pred, axis=1 ).mean(axis=1))
filename = 'test_pred-%i.pkl'%fold
outfile = open(filename,'wb')
 pickle.dump(arr_pred,outfile)
 outfile.close()
else:
for fold in range(KFOLD_SPLITS):
filename = '.. /input/dsamazinglyfastkernel09402/test_pred-%i.pkl'%fold
outfile = open(filename,'rb')
 arr_pred = pickle.load(outfile)
 arr_test_pred.append(np.stack(arr_pred, axis=1 ).mean(axis=1))
arr_test_pred_min.append(np.stack(arr_pred, axis=1 ).min(axis=1))
arr_test_pred_max.append(np.stack(arr_pred, axis=1 ).max(axis=1))
outfile.close() 
<load_from_csv>
file_1 = pd.read_csv(".. /input/melanoma-submission-9402/submission.csv")
 file_2 = pd.read_csv(".. /input/amazingly-fast-kernel/submission.csv")
 file_3 = pd.read_csv(".. /input/amazingly-fast-kernel-256/submission.csv")
 file_4 = pd.read_csv(".. /input/amazingly-fast-kernel-512/submission.csv")
 file_5 = pd.read_csv(".. /input/amazingly-fast-kernel-192/submission.csv")
 file_6 = pd.read_csv(".. /input/amazingly-fast-kernel-128/submission.csv")
 df = pd.DataFrame({"image_name":img_name_list_test})
 df["image_name"] = df["image_name"].map(lambda x: x.decode("utf-8"))
file_1 = pd.merge(df, file_1, on="image_name")
 file_2 = pd.merge(df, file_2, on="image_name")
 file_3 = pd.merge(df, file_3, on="image_name")
 file_4 = pd.merge(df, file_4, on="image_name")
 file_5 = pd.merge(df, file_5, on="image_name")
 file_6 = pd.merge(df, file_6, on="image_name")
 df["target_1"] = list(file_1["target"].values)
 df["target_2"] = list(file_2["target"].values)
 df["target_3"] = list(file_3["target"].values)
 df["target_4"] = list(file_4["target"].values)
 df["target_5"] = list(arr_test_pred[0][:,0])
 df["target_6"] = list(arr_test_pred[1][:,0])
 df["target_7"] = list(arr_test_pred[2][:,0])
 df["target_8"] = list(arr_test_pred[3][:,0])
 df["target_9"] = list(arr_test_pred[4][:,0])
 df["target_10"] = list(file_5["target"].values)
 df["target_11"] = list(file_6["target"].values)
<load_from_csv>
df_train_pop['Province.State'].fillna("None", inplace=True)
 df_train_pop['Country.Region'].fillna("None", inplace=True)
 df_train_pop['Population'] = df_train_pop['Population'].fillna(0 )
meta = pd.read_csv(".. /input/meta-data-model/submission.csv" )<feature_engineering>
missings_count = {col:df_train_pop[col].isnull().sum() for col in df_train_pop.columns}
missings = pd.DataFrame.from_dict(missings_count, orient='index')
 print(missings.nlargest(30, 0))
df["target"] = df.iloc[:,1:12].mean(axis=1)
 df["target_min"] = df.iloc[:,1:12].min(axis=1)
 df["target_max"] = df.iloc[:,1:12].max(axis=1 )<feature_engineering>
df_fin_selc = df_train_pop[['Province_State','Country_Region','Date','ConfirmedCases','Fatalities','Day_num','Day','Month','Year','Population']]
if 1==2:
cutoff_lo = 0.88
cutoff_hi = 0.11
df['target'] = np.where(np.all(df.iloc[:, 1:10] > cutoff_lo, axis=1),
df['target_max'],
np.where(np.all(df.iloc[:, 1:10] < cutoff_hi, axis=1),
df['target_min'],
df['target']))<merge>
df_fin_selc['I'] = df_fin_selc['ConfirmedCases']-df_fin_selc['Fatalities']
df_fin_selc['R'] = df_fin_selc['Fatalities']
df_fin_selc['S'] = df_fin_selc['Population']-df_fin_selc['ConfirmedCases']
df_SIR = df_fin_selc[['Province_State','Country_Region','Population','Day_num','S','I','R']].reindex()
df_SIR[df_SIR['Country_Region'] == 'Italy'].tail(5 )
if 1==2:
meta = meta[["image_name","target"]]
meta.columns = ["image_name","target_meta"]
df = df[["image_name","target"]]
df.columns = ["image_name","target_model"]
df = pd.merge(df[["image_name","target_model"]], meta[["image_name","target_meta"]], on="image_name")
 df["target"] = df.apply(lambda row: 0.9*row["target_model"] + 0.1*row["target_meta"], axis=1 )<feature_engineering>
import numpy as np
from scipy.integrate import odeint
from scipy import integrate, optimize
<save_to_csv>
def SIR_testmodel(y,t,bta,gmma):
S, I, R = y
dS_dt = -1*bta*I*S/N
dI_dt =(bta*I*S/N)- gmma*I
dR_dt = gmma*I
return([dS_dt, dI_dt, dR_dt] )
df[["image_name","target"]].to_csv("submission.csv", index=False )<install_modules>
N = 1
S00 = 0.9
I00 = 0.1
R00 = 0.0
bta = 0.35
gmma = 0.1
t = np.linspace(0,100,1000)
 sol = odeint(SIR_testmodel,[S00,I00,R00],t,args =(bta,gmma))
sol = np.array(sol )
!pip install -q efficientnet >> /dev/null<import_modules>
display(covid_pop_sel[covid_pop_sel['Country.Region']=='Italy'] )
import os
import gc
import random, re, math
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import tensorflow as tf
import tensorflow.keras.backend as K
from tensorflow.keras.losses import Loss
from tensorflow.keras.applications import ResNet101
from tensorflow.keras.layers import Input, Dense, GlobalAveragePooling2D, Lambda
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.optimizers import Adam
import efficientnet.tfkeras as efn
from sklearn.model_selection import KFold
from sklearn.utils import shuffle
from sklearn.preprocessing import OneHotEncoder
from kaggle_datasets import KaggleDatasets
import cv2<define_variables>
Country_Dates[Country_Dates['Country_Region'] == 'Italy']
AUTO = tf.data.experimental.AUTOTUNE
IMAGE_SIZE = [256, 256]
EPOCHS = 7
BATCH_SIZE = max(32 * strategy.num_replicas_in_sync, 64)
 TF_RECORDS_FILES = KaggleDatasets().get_gcs_path('512x512-melanoma-tfrecords-70k-images')
 TRAIN_CSV = ".. /input/siim-isic-melanoma-classification/train.csv"
TEST_CSV = ".. /input/siim-isic-melanoma-classification/test.csv"
SUBMISSION_CSV = "submission.csv"<choose_model_class>
cond1 = Italy_SIR_df.I >= 1
test = Italy_SIR_df['Day_num'][(cond1)]
test
Italy_SIR_df.iloc[:10]
LR_START = 0.000005
LR_MAX = 0.0004
LR_MIN = 0.000001
LR_RAMPUP_EPOCHS = 1
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY = 0.75
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)
 rng = [i for i in range(16)]
y = [lrfn(x)for x in rng]
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))<normalization>
Italy_S = Italy_SIR_df['S']
Italy_I = Italy_SIR_df['I']
Italy_R = Italy_SIR_df['R']
Italy_s = np.array(Italy_S, dtype=float)
 Italy_i = np.array(Italy_I, dtype=float)
 Italy_r = np.array(Italy_R, dtype=float)
 display(Italy_s[9],Italy_i[9], Italy_r[9] )
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])
 zoom_matrix = tf.reshape(tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3])
 shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])
 return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))
def transform(image):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
rot = 180.* tf.random.normal([1],dtype='float32')
 shr = 2.* tf.random.normal([1],dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1],dtype='float32')*0.12
w_zoom = 1.0 + tf.random.normal([1],dtype='float32')*0.12
h_shift = 8.* tf.random.normal([1],dtype='float32')
 w_shift = 8.* tf.random.normal([1],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image,tf.transpose(idx3))
im = tf.reshape(d,[DIM,DIM,3])
 im = tf.image.random_flip_left_right(im)
 im = tf.image.random_flip_up_down(im)
 im = tf.image.random_brightness(im, 0.2)
 im = tf.image.random_contrast(im, 0.9, 1.1)
 im = tf.image.random_hue(im, 0.05)
 return im
def transform_batch(images, feats, labels):
ret = tf.map_fn(transform, images)
 return ret, feats, labels 
<feature_engineering>
def SIR_model(y,t,beta,gamma):
S, I, R = y
N = 60480000
dS_dt = -1*beta*I*S/N
dI_dt =(beta*I*S/N)- gamma*I
dR_dt = gamma*I
return([dS_dt, dI_dt, dR_dt])
 def fit_odeint(x, beta, gamma):
return integrate.odeint(SIR_model,(S0, I0, R0), x, args=(beta, gamma)) [:,1]
train_feature_description = {
"image_name": tf.io.FixedLenFeature([], tf.string),
"image": tf.io.FixedLenFeature([], tf.string),
'age_approx': tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
"target": tf.io.FixedLenFeature([], tf.int64)
 }
test_feature_description = {
"image_name": tf.io.FixedLenFeature([], tf.string),
"image": tf.io.FixedLenFeature([], tf.string),
'age_approx': tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
}
def parse_example(W_LABEL):
def function(ex):
if W_LABEL:
feature_description = train_feature_description
else:
feature_description = test_feature_description
return tf.io.parse_single_example(ex, feature_description)
 return function
def decode_image(W_LABEL, IMAGE_SIZE):
def function(ex):
image = ex['image']
age =(ex['age_approx']-51)/(20)
 ana = tf.cast(tf.one_hot(ex['anatom_site_general_challenge'], 7), tf.float32)
 feats = tf.concat([ana, tf.cast(tf.expand_dims(age, 0), tf.float32)], 0)
 if W_LABEL:
label = ex['target']
feature_description = train_feature_description
else:
feature_description = test_feature_description
image_name = ex['image_name']
image = tf.image.decode_jpeg(image, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, IMAGE_SIZE)
 image = tf.reshape(image, [*IMAGE_SIZE, 3])
 ex['image'] = image
if W_LABEL:
return image, feats, label
return image, feats, image_name
return function
def increase_pos(images, feats, labels):
LEN = tf.size(labels)
 pos = tf.squeeze(tf.where(tf.equal(labels, 1)) , -1)
 neg = tf.squeeze(tf.where(tf.equal(labels, 0)) , -1)
 if tf.size(pos)== 0:
return images, labels
if tf.size(neg)== 0:
return images, labels
neg = tf.tile(neg, [1+int(((LEN)//(2*tf.shape(neg)[0])))], name='neg_tile')
 neg = neg[0:LEN//2]
pos = tf.tile(pos, multiples=[1 +(LEN)//(2*tf.size(pos)) ], name='pos_tile')
 pos = pos[0:LEN//2]
indices = tf.concat([pos, neg], 0)
 indices = tf.random.shuffle(indices)
 imgs = tf.gather(images, indices)
 lbls = tf.gather(labels, indices)
 fts = tf.gather(feats, indices)
 return imgs, fts, lbls
def input_to_dict(images, feats, labels):
return {'in_img': images, 'in_feats': feats}, labels
class DataLoader:
def __init__(self, IMAGE_SIZE, BATCH_SIZE, TFR_FILES, CSV_FILE,
CACHE=True, W_LABELS=False, SPLIT=1.0, FROM_END=False, VAL=False):
self.IMAGE_SIZE = IMAGE_SIZE
self.BATCH_SIZE = BATCH_SIZE
self.TFR_FILES = TFR_FILES
self.CSV_FILE = CSV_FILE
self.CACHE = CACHE
self.W_LABELS = W_LABELS
self.SPLIT = SPLIT
self.FROM_END = FROM_END
self.VAL = VAL
self.load_csv()
self.create_ds()
def load_csv(self):
pass
def create_ds(self):
ignore_order = tf.data.Options()
if self.W_LABELS:
ignore_order.experimental_deterministic = False
self.dataset = tf.data.TFRecordDataset(self.TFR_FILES, num_parallel_reads = AUTO)
 if self.CACHE:
self.dataset = self.dataset.with_options(ignore_order)
 self.dataset = self.dataset.map(parse_example(self.W_LABELS), num_parallel_calls=AUTO)
 self.dataset = self.dataset.map(decode_image(self.W_LABELS, self.IMAGE_SIZE), num_parallel_calls=AUTO)
 if self.W_LABELS:
self.dataset = self.dataset.repeat()
self.dataset = self.dataset.shuffle(1024)
 self.dataset = self.dataset.batch(self.BATCH_SIZE, self.W_LABELS)
 if self.W_LABELS and(not self.VAL):
self.dataset = self.dataset.map(transform_batch, num_parallel_calls=AUTO)
 self.dataset = self.dataset.map(input_to_dict, num_parallel_calls=AUTO)
 self.dataset = self.dataset.prefetch(AUTO)
 def get_dataset(self):
return self.dataset
def get_iterations(self):
return self.datasize // self.BATCH_SIZE
class TrainDataLoader(DataLoader):

 def __init__(self, IMAGE_SIZE, BATCH_SIZE, VAL=False, CACHE=True, TFR_FILES=None):
if TFR_FILES is None:
TFR_FILES = tf.io.gfile.glob(TF_RECORDS_FILES + '/train*.tfrec')
 if VAL:
TFR_FILES = TFR_FILES[-5:]
else:
TFR_FILES = TFR_FILES[0:-5]
super(TrainDataLoader, self ).__init__(IMAGE_SIZE, BATCH_SIZE, TFR_FILES, CSV_FILE=TRAIN_CSV,
CACHE=CACHE, W_LABELS=True, VAL= VAL)
 class TestDataLoader(DataLoader):
def __init__(self, IMAGE_SIZE, BATCH_SIZE):
TFR_FILES = tf.io.gfile.glob(TF_RECORDS_FILES + '/test*.tfrec')
 super(TestDataLoader, self ).__init__(IMAGE_SIZE, BATCH_SIZE, TFR_FILES, CSV_FILE=TEST_CSV,
CACHE=False, W_LABELS=False, SPLIT=1.0, FROM_END=False )<load_pretrained>
xdata = Italy_SIR_df.Day_num
ydata = Italy_i
xdata = np.array(xdata, dtype=float)
 S0 = 60479998.0
I0 = 2.0
R0 = 0
y = S0, I0, R0
popt, pcov = optimize.curve_fit(fit_odeint, xdata, ydata)
 fitted = fit_odeint(xdata, *popt )
train_dataset = TrainDataLoader(IMAGE_SIZE, BATCH_SIZE, False, True)
 val_dataset = TrainDataLoader(IMAGE_SIZE, BATCH_SIZE, True, True )<compute_test_metric>
comp_df = pd.read_csv('.. /input/corona-virus-report/covid_19_clean_complete.csv')
 comp_df.head(5 )
def loss_function(y_true, y_pred):
return tf.math.reduce_mean(-(y_true*tf.math.log(y_pred)*0.99 +(1-y_true)*tf.math.log(1-y_pred)*0.01))
def get_f1(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 recall = true_positives /(possible_positives + K.epsilon())
 f1_val = 2*(precision*recall)/(precision+recall+K.epsilon())
 return f1_val 
<define_variables>
comp_df['Province/State'].fillna("None", inplace=True)
 comp_df[comp_df.iloc[:,0:].isnull().any(axis = 1)].iloc[:,0:].head()
aucs, models = zip(*sorted(zip([hist.history['val_auc'][-1] for hist in hists], models)))
 models = list(models)
 aucs = list(aucs)
 print(aucs )<choose_model_class>
comp_df['Date'] = pd.to_datetime(comp_df['Date'] )
with strategy.scope() :
in_img = tf.keras.layers.Input(( *IMAGE_SIZE, 3), name='in_img')
 in_feats = tf.keras.layers.Input(( 8), name='in_feats')
 preds = []
for model in models:
model.trainable = False
temp_mod = Model(model.input, model.layers[-2].output)
 preds.append(temp_mod({'in_img': in_img, 'in_feats': in_feats}))
y = tf.keras.layers.Concatenate(axis=1 )(preds)
 y = tf.keras.layers.Dense(1, 'sigmoid' )(y)
 forest = Model([in_img, in_feats], y)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.01)
 forest.compile('adam', loss=loss, metrics=[
tf.keras.metrics.BinaryAccuracy(name='accuracy'),
tf.keras.metrics.Precision(name='precision'),
tf.keras.metrics.Recall(name='recall'),
get_f1, 'AUC'
])
 train_dataset = TrainDataLoader(IMAGE_SIZE, BATCH_SIZE//len(models), False, True, TFR_FILES)
 forest.fit(train_dataset.get_dataset() ,
steps_per_epoch=236*len(models),
epochs=5,
callbacks = [lr_callback]
)<predict_on_test>
covid_pop_sel.loc[covid_pop_sel['Country.Region']=='Korea, South', 'Country.Region'] = 'South Korea'
comp_df_pop = comp_df.merge(covid_pop_sel, left_on = ['Country/Region','Province/State'], right_on = ['Country.Region','Province.State'], how = 'left')
 comp_df_pop.tail(3 )
def pred_to_label(pred, hthresh=0.766, lthresh=0.233):
if pred < lthresh:
return 0.0
elif pred > hthresh:
return 1.0
else:
return pred
test_dataset = TestDataLoader(IMAGE_SIZE, BATCH_SIZE)
 row_list = []
row_list_threshed = []
for batch in test_dataset.get_dataset() :
res = forest.predict_on_batch(batch[0])
 preds = [[img_name.decode("utf-8"), pred[0]] for img_name, pred in zip(batch[1].numpy() , res)]
row_list = row_list + preds
preds_threshed = [[img_name.decode("utf-8"), pred_to_label(pred[0])] for img_name, pred in zip(batch[1].numpy() , res)]
row_list_threshed = row_list_threshed + preds_threshed
df = pd.DataFrame(row_list,
columns =['image_name', 'target'] ).sort_values(by='image_name')
 df.to_csv(SUBMISSION_CSV, index=False)
 dft = pd.DataFrame(row_list_threshed,
columns =['image_name', 'target'] ).sort_values(by='image_name')
 dft.to_csv('t'+SUBMISSION_CSV, index=False )<import_modules>
comp_pop_sel = comp_df_pop[['Province/State','Country/Region','Date','Confirmed','Deaths','Recovered','Population']]
import numpy as np
import pandas as pd<load_from_csv>
missings_count = {col:comp_pop_sel[col].isnull().sum() for col in comp_pop_sel.columns}
missings = pd.DataFrame.from_dict(missings_count, orient='index')
 print(missings.nlargest(30, 0))
sub = pd.read_csv('.. /input/lyft-prediction-files/submission.csv')
 sub<save_to_csv>
comp_pop_sel['Population'] = comp_pop_sel['Population'].fillna(0 )
sub.to_csv('submission.csv', index=False )<save_to_csv>
comp_df_Italy = comp_pop_sel[comp_pop_sel['Country/Region']=='Italy']
comp_df_Italy = comp_df_Italy.reset_index(drop = True)
 comp_df_Italy.tail(5 )
sub.to_csv('submission.csv', index=False )<set_options>
comp_df_Italy['Day_num'] = preprocessing.LabelEncoder().fit_transform(comp_df_Italy.Date)
 comp_df_Italy.tail(3 )
pd.options.display.max_columns=305<define_variables>
comp_df_Italy['R'] = comp_df_Italy['Deaths']+comp_df_Italy['Recovered']
comp_df_Italy['I'] = comp_df_Italy['Confirmed']- comp_df_Italy['R']
comp_df_Italy['S'] = comp_df_Italy['Population']-comp_df_Italy['I']-comp_df_Italy['R']
comp_df_Italy.tail(3 )
paths = [
".. /input/lyft-best-performing-public-kernels/lyft-ensembling-raster-sizes.csv",
".. /input/lyft-best-performing-public-kernels/lyft-prediction-with-multi-mode-confidence.csv",
]
weights = [0.4, 0.6]<define_variables>
comp_df_Italy_S = comp_df_Italy['S']
comp_df_Italy_I = comp_df_Italy['I']
comp_df_Italy_R = comp_df_Italy['R']
comp_df_Italy_s = np.array(comp_df_Italy_S, dtype=float)
 comp_df_Italy_i = np.array(comp_df_Italy_I, dtype=float)
 comp_df_Italy_r = np.array(comp_df_Italy_R, dtype=float)
 display(comp_df_Italy_s[9],comp_df_Italy_i[9], comp_df_Italy_r[9] )
conf_cols = np.array(["conf_0", "conf_1", "conf_2"] )<concatenate>
df_train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 df_test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 df_subm = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/submission.csv' )
xy_cols = [[],[],[]]
for i in range(50):
for j in range(3):
xy_cols[j].append(f"coord_x{j}{i}")
 xy_cols[j].append(f"coord_y{j}{i}")
 xy_cols[0][:10]<define_variables>
df_train_copy = df_train.copy()
df_test_copy = df_test.copy()
COLUMNS = ["timestamp", "track_id"] + list(conf_cols)+ xy_cols[0] + xy_cols[1] + xy_cols[2]<prepare_output>
df_train.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_train.rename(columns={'Province_State':'State'}, inplace=True)
 df_train['Date'] = pd.to_datetime(df_train['Date'], infer_datetime_format=True)
 df_test.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_test.rename(columns={'Province_State':'State'}, inplace=True)
 df_test['Date'] = pd.to_datetime(df_test['Date'], infer_datetime_format=True )
def sort_df(df, sort_timestamp_track_id=True):
conf_orders = np.argsort(-df[conf_cols].values,1)
 XY = np.stack([df[xy_cols[0]].values,df[xy_cols[1]].values, df[xy_cols[2]].values], axis=1)
 XY = XY[np.arange(len(XY)) [:, None], conf_orders]
df2 = pd.DataFrame(columns = COLUMNS)
 df2["timestamp"] = df["timestamp"].values
df2["track_id"] = df["track_id"].values
df2[xy_cols[0] + xy_cols[1] + xy_cols[2]] = XY.reshape(-1,300)
 df2[conf_cols] = df[conf_cols].values[np.arange(len(df)) [:, None], conf_orders]
if sort_timestamp_track_id:
df2.sort_values(["timestamp", "track_id"], inplace=True)
 df2.reset_index(inplace=True, drop=True)
 return df2<load_from_csv>
y1_df_Train = df_train.iloc[:, -2]
y2_df_Train = df_train.iloc[:, -1]
NOVAL = "NOVAL"
def handlenullstate(State, Country):
if State == NOVAL: return Country
return State
%%time
df = None
for path,w in zip(paths,weights):
print(w, path)
 temp = pd.read_csv(path)
 temp = sort_df(temp)
 temp[COLUMNS[5:]] *= w
if df is None:
df = temp
else:
df[COLUMNS[2:]] += temp[COLUMNS[2:]]
df[conf_cols] /= df[conf_cols].sum(1 ).values[:, None]
sample = pd.read_csv(".. /input/lyft-motion-prediction-autonomous-vehicles/multi_mode_sample_submission.csv")
 df = sample[["timestamp", "track_id"]].merge(df, on=["timestamp", "track_id"])
 sample.shape, df.shape<save_to_csv>
df_train['State'].fillna(NOVAL, inplace=True)
 df_train['State'] = df_train.loc[:, ['State', 'Country']].apply(lambda x : handlenullstate(x['State'], x['Country']), axis=1)
 df_train.loc[:, 'Date'] = df_train.Date.dt.strftime("%m%d")
 df_train["Date"] = df_train["Date"].astype(int)
 df_test['State'].fillna(NOVAL, inplace=True)
 df_test['State'] = df_test.loc[:, ['State', 'Country']].apply(lambda x : handlenullstate(x['State'], x['Country']), axis=1)
 df_test.loc[:, 'Date'] = df_test.Date.dt.strftime("%m%d")
 df_test["Date"] = df_test["Date"].astype(int)
 df_test.head()
df.to_csv("submission.csv", index=False, float_format='%.6f' )<set_options>
covid_pop_taken = covid_pop[['Province.State','Country.Region','Population']]
pd.options.display.max_columns=305<define_variables>
covid_pop_taken.rename(columns={'Province.State':'State'}, inplace=True)
 covid_pop_taken.rename(columns={'Country.Region':'Country'}, inplace=True )
paths = [
".. /input/lyft-best-performing-public-kernels/lyft-ensembling-raster-sizes.csv",
".. /input/lyft-best-performing-public-kernels/lyft-prediction-with-multi-mode-confidence.csv",
]
weights = [0.4, 0.6]<define_variables>
covid_pop_taken['State'].fillna("None", inplace=True)
 NOVAL = 'None'
covid_pop_taken['State'] = covid_pop_taken.loc[:, ['State', 'Country']].apply(lambda x : handlenullstate(x['State'], x['Country']), axis=1)
 covid_pop_taken.head(3 )
conf_cols = np.array(["conf_0", "conf_1", "conf_2"] )<concatenate>
df_train_copy2 = df_train.copy()
df_test_copy2 = df_test.copy()
df_test.head(3 )
xy_cols = [[],[],[]]
for i in range(50):
for j in range(3):
xy_cols[j].append(f"coord_x{j}{i}")
 xy_cols[j].append(f"coord_y{j}{i}")
 xy_cols[0][:10]<define_variables>
df_train_co_pop = df_train.merge(covid_pop_taken, left_on = ['Country','State'], right_on = ['Country','State'], how = 'left')
 df_test_co_pop = df_test.merge(covid_pop_taken, left_on = ['Country','State'], right_on = ['Country','State'], how = 'left')
 df_test_co_pop.head(5 )
COLUMNS = ["timestamp", "track_id"] + list(conf_cols)+ xy_cols[0] + xy_cols[1] + xy_cols[2]<prepare_output>
df_train_co_pop[['Population']] = preprocessing.scale(df_train_co_pop[['Population']])
 df_test_co_pop[['Population']] = preprocessing.scale(df_test_co_pop[['Population']] )
def sort_df(df, sort_timestamp_track_id=True):
conf_orders = np.argsort(-df[conf_cols].values,1)
 XY = np.stack([df[xy_cols[0]].values,df[xy_cols[1]].values, df[xy_cols[2]].values], axis=1)
 XY = XY[np.arange(len(XY)) [:, None], conf_orders]
df2 = pd.DataFrame(columns = COLUMNS)
 df2["timestamp"] = df["timestamp"].values
df2["track_id"] = df["track_id"].values
df2[xy_cols[0] + xy_cols[1] + xy_cols[2]] = XY.reshape(-1,300)
 df2[conf_cols] = df[conf_cols].values[np.arange(len(df)) [:, None], conf_orders]
if sort_timestamp_track_id:
df2.sort_values(["timestamp", "track_id"], inplace=True)
 df2.reset_index(inplace=True, drop=True)
 return df2<load_from_csv>
df_train = df_train_co_pop
df_test = df_test_co_pop
%%time
df = None
for path,w in zip(paths,weights):
print(w, path)
 temp = pd.read_csv(path)
 temp = sort_df(temp)
 temp[COLUMNS[5:]] *= w
if df is None:
df = temp
else:
df[COLUMNS[2:]] += temp[COLUMNS[2:]]
df[conf_cols] /= df[conf_cols].sum(1 ).values[:, None]
sample = pd.read_csv(".. /input/lyft-motion-prediction-autonomous-vehicles/multi_mode_sample_submission.csv")
 df = sample[["timestamp", "track_id"]].merge(df, on=["timestamp", "track_id"])
 sample.shape, df.shape<save_to_csv>
le = preprocessing.LabelEncoder()
df_train.Country = le.fit_transform(df_train.Country)
 df_train['State'] = le.fit_transform(df_train['State'])
 df_train.Country = le.fit_transform(df_train.Country)
 df_test.Country = le.fit_transform(df_test.Country)
 df_test['State'] = le.fit_transform(df_test['State'])
 df_test.head()
df.to_csv("submission.csv", index=False, float_format='%.6f' )<set_options>
filterwarnings('ignore')
 le = preprocessing.LabelEncoder()
countries = df_train.Country.unique()
pd.options.display.max_columns=305<define_variables>
xout = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 for country in countries:
states = df_train.loc[df_train.Country == country, :].State.unique()
for state in states:
x_train_CS = df_train.loc[(df_train.Country == country)&(df_train.State == state), ['State', 'Country', 'Date', 'ConfirmedCases', 'Fatalities','Population']]
y1_train_CS = x_train_CS.loc[:, 'ConfirmedCases']
y2_train_CS = x_train_CS.loc[:, 'Fatalities']
x_train_CS = x_train_CS.loc[:, ['State', 'Country', 'Date','Population']]
x_train_CS.Country = le.fit_transform(x_train_CS.Country)
 x_train_CS['State'] = le.fit_transform(x_train_CS['State'])
 x_test_CS = df_test.loc[(df_test.Country == country)&(df_test.State == state), ['State', 'Country', 'Date', 'ForecastId','Population']]
x_test_CS_Id = x_test_CS.loc[:, 'ForecastId']
x_test_CS = x_test_CS.loc[:, ['State', 'Country', 'Date','Population']]
x_test_CS.Country = le.fit_transform(x_test_CS.Country)
 x_test_CS['State'] = le.fit_transform(x_test_CS['State'])
 xmodel1 = XGBRegressor(n_estimators=1000)
 xmodel1.fit(x_train_CS, y1_train_CS)
 y1_xpred = xmodel1.predict(x_test_CS)
 xmodel2 = XGBRegressor(n_estimators=1000)
 xmodel2.fit(x_train_CS, y2_train_CS)
 y2_xpred = xmodel2.predict(x_test_CS)
 xdata = pd.DataFrame({'ForecastId': x_test_CS_Id, 'ConfirmedCases': y1_xpred, 'Fatalities': y2_xpred})
 xout = pd.concat([xout, xdata], axis=0)
paths = [
".. /input/lyft-best-performing-public-kernels/lyft-ensembling-raster-sizes.csv",
".. /input/lyft-best-performing-public-kernels/lyft-prediction-with-multi-mode-confidence.csv",
]
weights = [0.4, 0.6]<define_variables>
xout.ForecastId = xout.ForecastId.astype('int')
 xout['ConfirmedCases'] = round(xout['ConfirmedCases'],1)
 xout['Fatalities'] = round(xout['Fatalities'],1)
 display(xout.head() )
conf_cols = np.array(["conf_0", "conf_1", "conf_2"] )<concatenate>
xout['ConfirmedCases'] = xout['ConfirmedCases'].apply(int)
 xout['Fatalities'] = xout['Fatalities'].apply(int )
xy_cols = [[],[],[]]
for i in range(50):
for j in range(3):
xy_cols[j].append(f"coord_x{j}{i}")
 xy_cols[j].append(f"coord_y{j}{i}")
 xy_cols[0][:10]<define_variables>
xout = xout.drop_duplicates()
xout.reindex()
<prepare_output><EOS> 
xout.to_csv('submission.csv', index=False )