Kaggle competition: COVID19 Global Forecasting (Week 4)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-4)<normalization>
import numpy as np
import pandas as pd
import datetime as datetime
from sklearn import preprocessing
ROT_ = 180.0
SHR_ = 2.0
HZOOM_ = 8.0
WZOOM_ = 8.0
HSHIFT_ = 8.0
WSHIFT_ = 8.0
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, DIM=img_size):
XDIM = DIM%2
rot = ROT_ * tf.random.normal([1], dtype='float32')
 shr = SHR_ * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ HZOOM_
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ WZOOM_
h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32')
 w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )<prepare_x_and_y>
register_matplotlib_converters()
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 data = {}
image = prepare_image(example['image'])
 data['patient_id'] = tf.cast(example['patient_id'],tf.float32)
 data['sex'] = tf.cast(example['patient_id'],tf.float32)
 data['age_approx'] = tf.cast(example['age_approx'],tf.float32)
 data['anatom_site_general_challenge'] = tf.cast(example['anatom_site_general_challenge'],tf.float32)
 label = tf.cast(example['target'], tf.float32)
 return image, label , data
def read_unlabeled_tfrecord(example,augment ,return_image_name):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
"patient_id" : tf.io.FixedLenFeature([],tf.int64),
"sex": tf.io.FixedLenFeature([],tf.int64),
"age_approx": tf.io.FixedLenFeature([],tf.int64),
"anatom_site_general_challenge": tf.io.FixedLenFeature([],tf.int64),
}
example = tf.io.parse_single_example(example, tfrec_format)
 data = {}
image = prepare_image(example['image'],augment = augment)
 img_name = example['image_name'] if return_image_name else 0
data['patient_id'] = tf.cast(example['patient_id'],tf.float32)
 data['sex'] = tf.cast(example['patient_id'],tf.float32)
 data['age_approx'] = tf.cast(example['age_approx'],tf.float32)
 data['anatom_site_general_challenge'] = tf.cast(example['anatom_site_general_challenge'],tf.float32)
 return image, img_name , data
def prepare_image(img, augment=True, dim=img_size):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.cast(img, tf.float32)/ 255.0
if augment:
img = transform(img,DIM=dim)
 if(RATE!=0)&(CT!=0)&(SIZE!=0):
img = dropout(img, DIM=dim, PROBABILITY=RATE, CT=CT, SZ=SIZE)
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 img = tf.reshape(img, [dim,dim, 3])
 img = tf.image.resize(img, [img_size,img_size])
 return img
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )<categorify>
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestRegressor
from xgboost import XGBRegressor
def train_setup(image,label,data):
tab_data=[tf.cast(data[tfeat], dtype=tf.float32)for tfeat in ['sex','age_approx','patient_id','anatom_site_general_challenge']]
tabular=tf.stack(tab_data)
 return {'image_data' : image , 'meta_data' : tabular} , label
def test_setup(image,image_name,data):
tab_data=[tf.cast(data[tfeat], dtype=tf.float32)for tfeat in ['sex','age_approx','patient_id','anatom_site_general_challenge']]
tabular=tf.stack(tab_data)
 return {'image_data' : image , 'meta_data' : tabular } , image_name<create_dataframe>
training_set = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 training_set.shape
def get_dataset(files, augment = False, shuffle = False, repeat = False,
labeled=True, return_image_names=True,val = False, batch_size=32, dim=img_size):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 ds = ds.map(train_setup , num_parallel_calls = AUTO)
 ds = ds.batch(batch_size * REPLICAS)
 ds = ds.prefetch(AUTO)
 if val == False :
if cutmix_rate :
ds = ds.map(cutmix_v2,num_parallel_calls = AUTO)
 if gridmask_rate:
ds = ds.map(gridmask_v2, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example,augment, return_image_names),
num_parallel_calls=AUTO)
 ds = ds.map(test_setup , num_parallel_calls = AUTO)
 ds = ds.batch(batch_size * REPLICAS)
 ds = ds.prefetch(AUTO)
 return ds<choose_model_class>
test_set = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 test_set.shape
EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3,
efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6]
def build_model(dim=img_size, ef=0):
inp = tf.keras.layers.Input(shape=(dim,dim,3),name = 'image_data')
 inp2 = tf.keras.layers.Input(shape=(4),name = 'meta_data')
 base = EFNS[ef](input_shape=(dim,dim,3),weights='noisy-student',include_top=False , drop_connect_rate = 0.4)
 x = base(inp)
 x = tf.keras.layers.GlobalAveragePooling2D()(x)
 x2 = tf.keras.layers.Dense(128, activation='relu' )(inp2)
 concat = tf.keras.layers.concatenate([x,x2])
 concat = tf.keras.layers.Dense(512,activation = 'relu' )(concat)
 concat = tf.keras.layers.Dropout(0.4 )(concat)
 concat = tf.keras.layers.Dense(1,activation='sigmoid',dtype = 'float32' )(concat)
 model = tf.keras.Model(inputs=[inp,inp2],outputs=concat)
 opt = tf.keras.optimizers.Adam(learning_rate=0.001)
 if focal_loss :
loss = tfa.losses.SigmoidFocalCrossEntropy(reduction=tf.keras.losses.Reduction.AUTO)
 else :
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05)
 model.compile(optimizer=opt,loss=loss,metrics=['AUC'])
 return model<init_hyperparams>
my_country='Italy'
my_variable='ConfirmedCases'
def get_lr_callback(batch_size=8):
lr_start = 0.000005
lr_max = 0.00000125 * REPLICAS * batch_size
lr_min = 0.000001
lr_ramp_ep = 5
lr_sus_ep = 0
lr_decay = 0.8
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback<define_variables>
N_train=len(training_set[training_set['Country_Region']==my_country])
 N_train
VERBOSE = 0
DISPLAY_PLOT = True
skf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)
 oof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = []
preds = np.zeros(( count_data_items(files_test),1))
for fold,(idxT,idxV)in enumerate(skf.split(np.arange(15))):
if DEVICE=='TPU':
if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)
 print('
print(' 
(IMG_SIZES[fold],EFF_NETS[fold],BATCH_SIZES[fold]*REPLICAS))
files_train = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxT])
 if INC2019[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2+1])
 print('
if INC2018[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2])
 print('
if PSEUDO[fold] :
files_train += tf.io.gfile.glob(GCS_PATH3[fold]+'/train_pseudo01-323.tfrec')
 files_train += tf.io.gfile.glob(GCS_PATH3[fold]+'/train_pseudo00-1000.tfrec')
 print('
np.random.shuffle(files_train); print('
files_valid = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxV])
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[fold] + '/test*.tfrec')))
 K.clear_session()
with strategy.scope() :
model = build_model(dim=img_size,ef=EFF_NETS[fold])
 sv = tf.keras.callbacks.ModelCheckpoint(
 'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,
save_weights_only=True, mode='min', save_freq='epoch')
 print('Training...')
 history = model.fit(
 get_dataset(files_train, augment=True, shuffle=True, repeat=True,
dim=IMG_SIZES[fold], batch_size = BATCH_SIZES[fold]),
epochs=EPOCHS[fold], callbacks = [sv,get_lr_callback(BATCH_SIZES[fold])],
steps_per_epoch=count_data_items(files_train)/BATCH_SIZES[fold]//REPLICAS,
validation_data=get_dataset(files_valid,augment=False,shuffle=False,
repeat=False,dim=IMG_SIZES[fold],val=True),
verbose=VERBOSE 
)
 print('Loading best model...')
 model.load_weights('fold-%i.h5'%fold)
 print('Predicting OOF with TTA...')
 ds_valid = get_dataset(files_valid,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)
 ct_valid = count_data_items(files_valid); STEPS = TTA * ct_valid/BATCH_SIZES[fold]/4/REPLICAS
pred = model.predict(ds_valid,steps=STEPS,verbose=VERBOSE)[:TTA*ct_valid,]
oof_pred.append(np.mean(pred.reshape(( ct_valid,TTA),order='F'),axis=1))
ds_valid = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=True, return_image_names=True,val=True)
 oof_tar.append(np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]))
oof_folds.append(np.ones_like(oof_tar[-1],dtype='int8')*fold)
 ds = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 oof_names.append(np.array([img_name.numpy().decode("utf-8")for img, img_name in iter(ds.unbatch())]))
print('Predicting Test with TTA...')
 ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)
 ct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZES[fold]/4/REPLICAS
pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,]
preds[:,0] += np.mean(pred.reshape(( ct_test,TTA),order='F'),axis=1)* WGTS[fold]
auc = roc_auc_score(oof_tar[-1],oof_pred[-1])
 oof_val.append(np.max(history.history['val_auc']))
print('
if DISPLAY_PLOT:
plt.figure(figsize=(15,5))
plt.plot(np.arange(EPOCHS[fold]),history.history['auc'],'-o',label='Train AUC',color='
plt.plot(np.arange(EPOCHS[fold]),history.history['val_auc'],'-o',label='Val AUC',color='
x = np.argmax(history.history['val_auc']); y = np.max(history.history['val_auc'])
 xdist = plt.xlim() [1] - plt.xlim() [0]; ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
%.2f'%y,size=14)
 plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)
 plt.legend(loc=2)
 plt2 = plt.gca().twinx()
plt2.plot(np.arange(EPOCHS[fold]),history.history['loss'],'-o',label='Train Loss',color='
plt2.plot(np.arange(EPOCHS[fold]),history.history['val_loss'],'-o',label='Val Loss',color='
x = np.argmin(history.history['val_loss']); y = np.min(history.history['val_loss'])
 ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
plt.ylabel('Loss',size=14)
 plt.title('FOLD %i - Image Size %i, EfficientNet B%i, inc2019=%i, inc2018=%i'% 
(fold+1,IMG_SIZES[fold],EFF_NETS[fold],INC2019[fold],INC2018[fold]),size=18)
 plt.legend(loc=3)
 plt.show()<save_to_csv>
N_test=len(test_set[test_set['Country_Region']==my_country])
 N_test
oof = np.concatenate(oof_pred); true = np.concatenate(oof_tar);
names = np.concatenate(oof_names); folds = np.concatenate(oof_folds)
 auc = roc_auc_score(true,oof)
 print('Overall OOF AUC with TTA = %.3f'%auc)
 df_oof = pd.DataFrame(dict(
 image_name = names, target=true, pred = oof, fold=folds))
df_oof.to_csv('oof.csv',index=False)
 df_oof.head()<create_dataframe>
df_test = test_set.copy()
df_test['DsType']='test'
df_test['ConfirmedCases']=0
df_test['Fatalities']=0
df_test.info()
ds = get_dataset(files_test, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())] )<save_to_csv>
df_union=pd.concat([df_train,df_test],sort=False ).copy()
df_union.fillna('ND', inplace = True )
submission = pd.DataFrame(dict(image_name=image_names, target=preds[:,0]))
submission = submission.sort_values('image_name')
 submission.to_csv('stratify_e4_128meta_512.4concat_18epochs_512img_with_pseudo_labeling_noisy_0.3gridmask_cutmix0.4drop_connect.csv', index=False)
 submission.head()<load_from_csv>
df_union['Month']=df_union['Date'].apply(lambda s : int(s.replace('-','')[4:6]))
df_union['Day']=df_union['Date'].apply(lambda s : int(s.replace('-','')[6:9]))
df_union['Date']=df_union['Date'].apply(lambda s : datetime.datetime.strptime(s, '%Y-%m-%d'))
df_union['DateOrd']=df_union['Date'].apply(lambda s : s.toordinal())
 df_union['Province_Norm']=df_union['Country_Region']+'-'+df_union['Province_State']
df_union.tail()
top = -100
bottom = 2000
model_1 = pd.read_csv('.. /input/data-for-pseudolabelling-09576-09526/averaging_bestsub_withmeta_data_0.75_0.25giba.csv')
 model_2 = pd.read_csv('.. /input/data-for-pseudolabelling-09576-09526/averaging_bestsub_withmeta_data_0.7_0.3giba.csv')
 test = pd.read_csv('.. /input/siim-isic-melanoma-classification/test.csv')
 model_1.head(5 )<sort_values>
le1 = preprocessing.LabelEncoder()
le1.fit(df_union['Country_Region'])
 df_union['Country'] = le1.transform(df_union['Country_Region'] )
model1 = model_1.sort_values(by = ['target'])[top:].image_name
model1 = model1.values.tolist()
model2 = model_2.sort_values(by = ['target'])[top:].image_name
model2 = model2.values.tolist()
model3 = model_3.sort_values(by = ['target'])[top:].image_name
model3 = model3.values.tolist()
model4 = model_4.sort_values(by = ['target'])[top:].image_name
model4 = model4.values.tolist()
model5 = model_5.sort_values(by = ['target'])[top:].image_name
model5 = model5.values.tolist()<split>
le2 = preprocessing.LabelEncoder()
le2.fit(df_union['Province_Norm'])
 df_union['Province'] = le2.transform(df_union['Province_Norm'])
 df_union.drop('Province_Norm',axis=1,inplace=True )
label_1_imagename = set(model1 ).intersection(model2)
 print('Number of new positive examples: %i' %len(label_1_imagename))<feature_engineering>
def cut_neg(y):
m = max(0,np.amin(y))
for i in range(0,len(y)) :
if y[i]<m : y[i]=m
return y
label_1_data = test.loc[test['image_name'].isin(label_1_imagename)].reset_index(drop = True)
 label_1_data['target'] = 1
label_1_data.head(10 )<sort_values>
def linear_trend(X_train,y_true,X_test):
lrm = LinearRegression()
lrm.fit(X_train,y_true)
 y_valid = lrm.predict(X_train)
 y_pred = lrm.predict(X_test)
 y_valid = cut_neg(y_valid)
 y_pred = cut_neg(y_pred)
 return y_valid,y_pred
model1 = model_1.sort_values(by = ['target'])[:bottom].image_name
model1 = model1.values.tolist()
model2 = model_2.sort_values(by = ['target'])[:bottom].image_name
model2 = model2.values.tolist()
model3 = model_3.sort_values(by = ['target'])[:bottom].image_name
model3 = model3.values.tolist()
model4 = model_4.sort_values(by = ['target'])[:bottom].image_name
model4 = model4.values.tolist()
model5 = model_5.sort_values(by = ['target'])[:bottom].image_name
model5 = model5.values.tolist()<feature_engineering>
def polynomial_trend(deg,X_train,y_true,X_test):
pf = PolynomialFeatures(degree=deg)
 pr = pf.fit_transform(X_train)
 lrm = LinearRegression()
lrm.fit(pr, y_true)
 y_valid = lrm.predict(pf.fit_transform(X_train))
y_pred = lrm.predict(pf.fit_transform(X_test))
y_valid = cut_neg(y_valid)
 y_pred = cut_neg(y_pred)
 return y_valid,y_pred
label_0_data = test.loc[test['image_name'].isin(label_0_imagename)].reset_index(drop = True)
 label_0_data['target'] = 0
label_0_data.head(10 )<drop_column>
df_train = df_union[df_union['DsType']=='train'].drop('DsType',axis=1)
 df_test = df_union[df_union['DsType']=='test'].drop('DsType',axis=1)
 df_train['ConfirmedCasesValid'] = 0
df_train['FatalitiesValid'] = 0
df_train['ConfirmedCasesTrend'] = 0
df_train['FatalitiesTrend'] = 0
df_train['ConfirmedCasesResid'] = 0
df_train['FatalitiesResid'] = 0
df_test['ConfirmedCases'] = 0
df_test['Fatalities'] = 0
df_test['ConfirmedCasesTrend'] = 0
df_test['FatalitiesTrend'] = 0
df_test['ConfirmedCasesResid'] = 0
df_test['FatalitiesResid'] = 0
label_1_data.dropna(inplace = True)
 label_0_data.dropna(inplace = True )<concatenate>
y1_train = df_train['ConfirmedCases'].astype(float)
 y2_train = df_train['Fatalities'].astype(float )
data = pd.concat([label_0_data, label_1_data])
 data.shape<define_variables>
NPT = 4
md1 = RandomForestRegressor(random_state=1234)
 md2 = RandomForestRegressor(random_state=1234)
 for country in df_train['Country_Region'].unique() :
df_train_cy = df_train[df_train['Country_Region']==country].copy()
df_test_cy = df_test[df_test['Country_Region']==country].copy()
for province in df_train_cy['Province_State'].unique() :
print(f'Analizing > country = {country} : province = {province}')
 df_train_pr = df_train_cy[df_train_cy['Province_State']==province].copy()
df_test_pr = df_test_cy[df_test_cy['Province_State']==province].copy()
X_train_pr = df_train_pr[['DateOrd']]
y1_train_pr = df_train_pr['ConfirmedCases']
y2_train_pr = df_train_pr['Fatalities']
df_test_pr = df_test_pr[df_test_pr['Province_State']==province].copy()
X_test_pr = df_test_pr[['DateOrd']]
y1_check_pr_trend, y1_pred_pr_trend = polynomial_trend(NPT,X_train_pr,y1_train_pr,X_test_pr)
 y2_check_pr_trend, y2_pred_pr_trend = polynomial_trend(NPT,X_train_pr,y2_train_pr,X_test_pr)
 df_train.loc[(( df_train['Country_Region']==country)&(df_train['Province_State']==province)) ,'ConfirmedCasesTrend'] = y1_check_pr_trend
df_train.loc[(( df_train['Country_Region']==country)&(df_train['Province_State']==province)) ,'FatalitiesTrend'] = y2_check_pr_trend
df_test.loc[(( df_test['Country_Region']==country)&(df_test['Province_State']==province)) ,'ConfirmedCasesTrend'] = y1_pred_pr_trend
df_test.loc[(( df_test['Country_Region']==country)&(df_test['Province_State']==province)) ,'FatalitiesTrend'] = y2_pred_pr_trend
y1_train_pr_resid = y1_train_pr - y1_check_pr_trend
y2_train_pr_resid = y2_train_pr - y2_check_pr_trend
md1.fit(X_train_pr,y1_train_pr_resid)
 md2.fit(X_train_pr,y2_train_pr_resid)
 y1_check_pr_resid = md1.predict(X_train_pr)
 y2_check_pr_resid = md2.predict(X_train_pr)
 y1_pred_pr_resid = md1.predict(X_test_pr)
 y2_pred_pr_resid = md2.predict(X_test_pr)
 df_train.loc[(( df_train['Country_Region']==country)&(df_train['Province_State']==province)) ,'ConfirmedCasesResid'] = y1_train_pr_resid
df_train.loc[(( df_train['Country_Region']==country)&(df_train['Province_State']==province)) ,'FatalitiesResid'] = y2_train_pr_resid
df_test.loc[(( df_test['Country_Region']==country)&(df_test['Province_State']==province)) ,'ConfirmedCasesResid'] = y1_pred_pr_resid
df_test.loc[(( df_test['Country_Region']==country)&(df_test['Province_State']==province)) ,'FatalitiesResid'] = y2_pred_pr_resid
df_train.loc[(( df_train['Country_Region']==country)&(df_train['Province_State']==province)) ,'ConfirmedCasesValid'] = y1_check_pr_trend + y1_check_pr_resid
df_train.loc[(( df_train['Country_Region']==country)&(df_train['Province_State']==province)) ,'FatalitiesValid'] = y2_check_pr_trend + y2_check_pr_resid
df_test.loc[(( df_test['Country_Region']==country)&(df_test['Province_State']==province)) ,'ConfirmedCases'] = y1_pred_pr_trend + y1_pred_pr_resid
df_test.loc[(( df_test['Country_Region']==country)&(df_test['Province_State']==province)) ,'Fatalities'] = y2_pred_pr_trend + y2_pred_pr_resid
PATH = '.. /input/jpeg-melanoma-512x512/test/'
IMGS = data['image_name'].values
IMGS = IMGS + '.jpg'
print('Number of images: %i' %len(IMGS))<data_type_conversions>
from sklearn.metrics import mean_squared_log_error
str_col = ['patient_id','sex','anatom_site_general_challenge']
for col in str_col:
data[col], mp = data[col].factorize()
print(mp)
 data['age_approx'] = data['age_approx'].astype('int' )<feature_engineering>
def plotExample() :
train_cond =(( df_train['Country_Region']==my_country)&(df_train['Province_State']==my_province))
test_cond =(( df_test['Country_Region']==my_country)&(df_test['Province_State']==my_province))
x_test_plt = df_test[test_cond]['Date']
y_test_plt = df_test[test_cond][my_variable]
x_train_plt = df_train[train_cond]['Date']
y_train_plt = df_train[train_cond][my_variable]
y_valid_plt = df_train[train_cond][my_variable+'Valid']
plt.rcParams["figure.figsize"] =(12,6)
 fig, ax = plt.subplots()
ax.plot(x_train_plt,y_train_plt,'o')
 ax.plot(x_train_plt,y_valid_plt,'x')
 ax.plot(x_test_plt,y_test_plt,'*')
 ax.set_xticks([] )
def _bytes_feature(value):

 if isinstance(value, type(tf.constant(0))):
value = value.numpy()
return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
def _float_feature(value):

 return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))
def _int64_feature(value):

 return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))<categorify>
my_country='Italy'
my_province='ND'
my_variable='ConfirmedCases'
plotExample()
def serialize_example(feature0, feature1, feature2, feature3, feature4, feature5, feature6):
feature = {
'image': _bytes_feature(feature0),
'image_name': _bytes_feature(feature1),
'patient_id': _int64_feature(feature2),
'sex': _int64_feature(feature3),
'age_approx': _int64_feature(feature4),
'anatom_site_general_challenge': _int64_feature(feature5),
'target': _int64_feature(feature6)
 }
example_proto = tf.train.Example(features = tf.train.Features(feature = feature))
return example_proto.SerializeToString()<save_to_csv>
y1_valid = cut_neg(df_train['ConfirmedCasesValid'].copy())
 y2_valid = cut_neg(df_train['FatalitiesValid'].copy())
 y1_pred = cut_neg(df_test['ConfirmedCases'].copy())
 y2_pred = cut_neg(df_test['Fatalities'].copy() )
SIZE = 1000
CT = len(IMGS)//SIZE + int(len(IMGS)%SIZE != 0)
 for j in range(CT):
print() ; print('Writing TFRecord %i of %i...' %(j, CT))
CT2 = min(SIZE, len(IMGS)- j*SIZE)
 with tf.io.TFRecordWriter('train_pseudo_512%.2i-%i.tfrec' %(j, CT2)) as writer:
for k in range(CT2):
img = cv2.imread(PATH + IMGS[SIZE*j + k])
 img = cv2.imencode('.jpg', img,(cv2.IMWRITE_JPEG_QUALITY, 95)) [1].tostring()
name = IMGS[SIZE*j + k].split('.')[0]
row = data.loc[data.image_name == name]
example = serialize_example(
 img,
str.encode(name),
row.patient_id.values[0],
row.sex.values[0],
row.age_approx.values[0],
row.anatom_site_general_challenge.values[0],
row.target.values[0])
 writer.write(example)
 if k%100 == 0: print(k, ', ', end = '' )<load_from_csv>
mse =(mean_squared_log_error(y1_train, y1_valid)+mean_squared_log_error(y2_train, y2_valid)) /2
mse
train = pd.read_csv('.. /input/siim-isic-melanoma-classification/train.csv')
 print('Examples WITH Melanoma')
 imgs = train.loc[train.target==1].sample(10 ).image_name.values
plt.figure(figsize=(20,8))
for i,k in enumerate(imgs):
img = cv2.imread('.. /input/jpeg-melanoma-128x128/train/%s.jpg'%k)
 img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
 plt.subplot(2,5,i+1); plt.axis('off')
 plt.imshow(img)
 plt.show()
print('Examples WITHOUT Melanoma')
 imgs = train.loc[train.target==0].sample(10 ).image_name.values
plt.figure(figsize=(20,8))
for i,k in enumerate(imgs):
img = cv2.imread('.. /input/jpeg-melanoma-128x128/train/%s.jpg'%k)
 img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
 plt.subplot(2,5,i+1); plt.axis('off')
 plt.imshow(img)
 plt.show()<install_modules>
submission = pd.DataFrame({'ForecastId': df_test['ForecastId'],'ConfirmedCases': y1_pred,'Fatalities': y2_pred})
 submission.head()
<import_modules><EOS> 
submission.to_csv('submission.csv',index = False )