Kaggle competition: COVID19 Global Forecasting (Week 4)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-4)<normalization>
%matplotlib inline
plot_transform(7 )<define_variables>
test = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 train = pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 test = test[test.Date > "2020-04-14"]
all_data = pd.concat([train, test],ignore_index=True ).sort_values(by=['Country_Region','Province_State','Date'])
 all_data['ConfirmedCases'] = all_data['ConfirmedCases'].fillna(-1)
 all_data['Fatalities'] = all_data['Fatalities'].fillna(-0)
 countries = pd.read_csv(".. /input/countries/countries.csv")
 countries.drop(['iso_alpha3','iso_numeric','official_name','name','iso_alpha2'], axis='columns',inplace=True)
 data_quarantine = pd.read_csv(".. /input/countries/quarantine_dates.csv")
 data_quarantine = data_quarantine.groupby("Country" ).max().loc[:,"Start date"]
data_quarantine.loc["Russia"] = "2020-03-30"
countries = countries.set_index("ccse_name", drop=True)
 countries["Quarantine"] = data_quarantine
countries_mean = countries.mean()
all_data = all_data.merge(countries, how ="left" , left_on="Country_Region", right_on='ccse_name')
 all_data['fertility_rate'] = all_data['fertility_rate'].fillna(countries_mean['fertility_rate'])
 all_data['median_age'] = all_data['median_age'].fillna(countries_mean['median_age'])
 all_data['migrants'] = all_data['migrants'].fillna(countries_mean['migrants'])
 all_data['urban_pop_rate'] = all_data['urban_pop_rate'].fillna(countries_mean['urban_pop_rate'])
 all_data['density'] = all_data['density'].fillna(countries_mean['density'])
 all_data['land_area'] = all_data['land_area'].fillna(countries_mean['land_area'])
 all_data['population'] = all_data['population'].fillna(countries_mean['population'])
 all_data['world_share'] = all_data['world_share'].fillna(countries_mean['world_share'])
 all_data['Quarantine'] = all_data['Quarantine'].fillna("2020-04-01")
 all_data.drop(['Id'], axis='columns',inplace=True)
 all_data['Province_State'] = all_data['Province_State'].fillna("zzz" )
train_dataset =(load_dataset(train_filenames, labeled=True)
.map(data_augment, num_parallel_calls=AUTO)
.shuffle(SEED)
.batch(BATCH_SIZE,drop_remainder=True)
.repeat() 
.prefetch(AUTO))
valid_dataset =(load_dataset(valid_filenames, labeled=True)
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO))<choose_model_class>
data2 = all_data
data2 = data2[data2.ConfirmedCases != 0]
data2.loc[data2.ConfirmedCases == -1,"ConfirmedCases"] = 0
data2["Date"] = pd.to_datetime(data2.Date)
 data4 = data2[["Country_Region","Date"]].groupby("Country_Region" ).min()
data4.columns = ["Date_min"]
data2 = data2.merge(data4, how = 'left', left_on='Country_Region', right_on='Country_Region')
 data2["days"] =(data2.Date - data2.Date_min ).dt.days
data2["days_mart"] =(data2.Date - pd.to_datetime("2020-03-1")).dt.days
data2["days_after_Quarantine"] =(data2.Date - pd.to_datetime(data2.Quarantine)).dt.days
data2.drop(['Date_min'], axis='columns',inplace=True)
 data2.Date = data2["Date"].apply(lambda x: pd.Series(x.strftime("%m-%d")))
 data2 = data2.rename(columns={"ConfirmedCases": "confirmed","Fatalities":"deaths","Country_Region":"countries"})
 data2[data2.countries == "Russia"].iloc[0:70]
with strategy.scope() :
model = tf.keras.Sequential([
efn.EfficientNetB6(input_shape=(*SIZE, 3),weights='imagenet',pooling='avg',include_top=False),
Dense(1, activation='sigmoid')
 ])
 model.compile(
 optimizer='adam',
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = LABEL_SMOOTHING),
metrics=['accuracy',tf.keras.metrics.AUC(name='auc')] )<train_model>
data2[data2.countries == "Russia"]
if use_knockknock == True:
@telegram_sender(token=token, chat_id=int(chat_id))
def train() :
STEPS_PER_EPOCH = count_data_items(train_filenames)// BATCH_SIZE
history = model.fit(
 train_dataset,
epochs=EPOCHS,
callbacks=[lr_schedule],
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=valid_dataset)
 string = 'Train acc:{:.4f} Train loss:{:.4f} AUC: {:.4f}, Val acc:{:.4f} Val loss:{:.4f} Val AUC: {:.4f}'.format(\
model.history.history['accuracy'][-1],model.history.history['loss'][-1],\
model.history.history['auc'][-1],\
model.history.history['val_accuracy'][-1],model.history.history['val_loss'][-1],\
model.history.history['val_auc'][-1])
 return string
else:
def train() :
STEPS_PER_EPOCH = count_data_items(train_filenames)// BATCH_SIZE
history = model.fit(
 train_dataset,
epochs=EPOCHS,
callbacks=[lr_schedule],
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=valid_dataset)
 return<train_model>
data2.confirmed = np.log10(data2.confirmed+1)
 data2.deaths = np.log10(data2.deaths+1)
 old_con = data2["confirmed"].iloc[:-1]
old_con2 = data2["deaths"].iloc[:-1]
data2 = data2.iloc[1:]
data2["pred_conf"] = old_con.values
data2["pred_deaths"] = old_con2.values
data2 = data2.iloc[1:]
data2["delta_conf1"] = old_con.values[1:] - old_con.values[:-1]
data2["delta_deaths1"] = old_con2.values[1:] - old_con2.values[:-1]
data2 = data2.iloc[1:]
data2["delta_conf2"] = old_con.values[1:-1] - old_con.values[:-2]
data2["delta_deaths2"] = old_con2.values[1:-1] - old_con2.values[:-2]
data2 = data2.iloc[1:]
data2["delta_conf3"] = old_con.values[1:-2] - old_con.values[:-3]
data2["delta_deaths3"] = old_con2.values[1:-2] - old_con2.values[:-3]
data2 = data2.iloc[1:]
data2["delta_conf4"] = old_con.values[1:-3] - old_con.values[:-4]
data2["delta_deaths4"] = old_con2.values[1:-3] - old_con2.values[:-4]
data2.confirmed = data2.confirmed - data2["pred_conf"]
data2.deaths = data2.deaths - data2["pred_deaths"]
data2[data2.countries == "Russia"].iloc[0:70]
train()<load_from_csv>
days_x = 45
model_Confirmed, model_Death = {}, {}
data3 = data2.drop(['Quarantine','world_share', 'migrants','median_age','land_area','fertility_rate','density','Date'], axis='columns')
 data3.population = data3.population/10**6
one_hot = pd.get_dummies(data3['Province_State'])
 data3 = data3.join(one_hot)
 data_Korea = data3[(data3.countries == 'Korea, South')&(data3.confirmed > 0)]
data_Russia = data3[(data3.countries == 'Russia')&(data3.confirmed > 0)]
new_data = data3[data2.days_mart >= days_x]
old_data = data3[data2.days_mart < days_x]
train_labels = old_data.confirmed
train_death = old_data.deaths
train_data = old_data.drop(['Province_State', 'ForecastId','confirmed','countries','deaths'], axis='columns')
 train_data
num_test_images = count_data_items(test_filenames)
 submission_df = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')
 for i in range(TTA):
test_dataset =(load_dataset(test_filenames, labeled=False,ordered=True)
.map(data_augment, num_parallel_calls=AUTO)
.batch(BATCH_SIZE))
test_dataset_images = test_dataset.map(lambda image, image_name: image)
 test_dataset_image_name = test_dataset.map(lambda image, image_name: image_name ).unbatch()
test_ids = next(iter(test_dataset_image_name.batch(num_test_images)) ).numpy().astype('U')
 test_pred = model.predict(test_dataset_images, verbose=1)
 pred_df = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(test_pred)})
 temp = submission_df.copy()
del temp['target']
submission_df['target'] += temp.merge(pred_df,on="image_name")['target']/TTA<save_to_csv>
def pred_score(models, data, death = False, plot = 0):
predictions_all = 0
plt.rcParams['figure.figsize'] = [20, len(models)*10]
data = data[(data.confirmed > 0)]
death_labels = 10**(data.deaths + data.pred_deaths)- 1
labels = 10**(data.confirmed + data.pred_conf)- 1
data = data.drop(['Province_State','ForecastId','confirmed','countries','deaths'], axis='columns')
 for i, model in enumerate(models):
if model == "reg":
predictions = models[model].predict(scaler.transform(data))
else:
predictions = models[model].predict(data)
 if death:
predictions = predictions + data.pred_deaths.values
else:
predictions = predictions + data.pred_conf.values
predictions = 10**predictions - 1
predictions_all = predictions_all + predictions
if death:
print("Ошибка MALE по смертям", model, np.mean(np.abs(np.log10(( predictions+1)/(death_labels+1)))))
 else:
print("Ошибка MALE по заражениям", model, np.mean(np.abs(np.log10(( predictions+1)/(labels+1)))))
 if plot:
plt.subplot(len(models), 1, i+1)
 if plot==1:
plt.plot(predictions, label = "Предсказанное значение")
 if death:
plt.plot(death_labels.values, label = "Истинное значение")
 plt.gca().set(xlabel='Дни от случая первого заражения', ylabel='Смерти')
 else:
plt.plot(labels.values, label = "Истинное значение")
 plt.gca().set(xlabel='Дни от случая первого заражения', ylabel='Заражения')
 if plot==2:
plt.scatter(np.arange(0,len(predictions),1), predictions, s = 1, label = "Предсказанное значение")
 if death:
plt.scatter(np.arange(0,len(predictions),1), death_labels.values, s = 1, label = "Истинное значение")
 plt.gca().set(xlabel='Дни от случая первого заражения', ylabel='Смерти')
 else:
plt.scatter(np.arange(0,len(predictions),1), labels.values, s = 1, label = "Истинное значение")
 plt.gca().set(xlabel='Дни от случая первого заражения', ylabel='Заражения')
 plt.title(model)
 plt.grid(True)
 plt.legend()
predictions_all = predictions_all/len(models)
 if death:
print("Ошибка MALE по смертям средняя", np.mean(np.abs(np.log10(( predictions_all+1)/(death_labels+1)))))
 else:
print("Ошибка MALE по заражениям средняя", np.mean(np.abs(np.log10(( predictions_all+1)/(labels+1)))) )
submission_df.to_csv('submission.csv', index=False)
 pd.Series(np.round(submission_df['target'].values)).value_counts()<import_modules>
treeDepth = 30
mdl = tree.DecisionTreeRegressor(max_depth=treeDepth)
 param_grid = {
'n_estimators': [100],
'learning_rate': [0.0002],
'loss' : ["exponential"]
}
regrMdl = ensemble.AdaBoostRegressor(base_estimator=mdl)
 model_Confirmed["Adaboost"] = model_selection.RandomizedSearchCV(estimator = regrMdl, param_distributions = param_grid, n_iter = 100,
cv = 3, verbose=0, random_state=42, n_jobs = -1 ).fit(train_data, train_labels)
 model_Death["Adaboost"] = model_selection.RandomizedSearchCV(estimator = regrMdl, param_distributions = param_grid, n_iter = 100,
cv = 3, verbose=0, random_state=42, n_jobs = -1 ).fit(train_data, train_death )
!pip install -q efficientnet 
<set_options>
model_Confirmed["RandomForest"] = ensemble.RandomForestRegressor(n_estimators=200, max_depth=30, random_state=42,
n_jobs = -1 ).fit(train_data, train_labels)
 model_Death["RandomForest"] = ensemble.RandomForestRegressor(n_estimators=200, max_depth=30, random_state=42,
n_jobs = -1 ).fit(train_data, train_death)
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<load_from_csv>
model_Confirmed["Xgboost"] = xgb.XGBRegressor(objective ='reg:squarederror',n_estimators=200, max_depth=20, random_state=42,
n_jobs = -1 ).fit(train_data, train_labels)
 model_Death["Xgboost"] = xgb.XGBRegressor(objective ='reg:squarederror',n_estimators=200, max_depth=20, random_state=42,
n_jobs = -1 ).fit(train_data, train_death )
AUTO = tf.data.experimental.AUTOTUNE
GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-384x384')
 EPOCHS = 40
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
AUG_BATCH = BATCH_SIZE
IMAGE_SIZE = [384, 384]
SEED = 123
LR = 0.0003
cutmix_rate = 0.30
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')
 SUB = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv' )<normalization>
pred_score(model, data_pred, death_bool, plot_type )
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])
 zoom_matrix = tf.reshape(tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3])
 shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])
 return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))
def transform(image, label):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
tmp = random.uniform(0, 1)
 if 0 < tmp <= 0.1:
rot = 15.0 * tf.random.normal([1],dtype='float32')
 elif 0.1 < tmp <= 0.2:
rot = 30.0 * tf.random.normal([1],dtype='float32')
 elif 0.2 < tmp <= 0.3:
rot = 45.0 * tf.random.normal([1],dtype='float32')
 elif 0.3 < tmp <= 0.4:
rot = 60.0 * tf.random.normal([1],dtype='float32')
 elif 0.4 < tmp <= 0.5:
rot = 75.0 * tf.random.normal([1],dtype='float32')
 elif 0.5 < tmp <= 0.6:
rot = 90.0 * tf.random.normal([1],dtype='float32')
 elif 0.6 < tmp <= 0.7:
rot = 110.0 * tf.random.normal([1],dtype='float32')
 elif 0.7 < tmp <= 0.8:
rot = 130.0 * tf.random.normal([1],dtype='float32')
 elif 0.8 < tmp <= 0.9:
rot = 150.0 * tf.random.normal([1],dtype='float32')
 elif 0.9 < tmp <= 1.0:
rot = 180.0 * tf.random.normal([1],dtype='float32')
 shr = 5.* tf.random.normal([1],dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.
 h_shift = 16.* tf.random.normal([1],dtype='float32')
 w_shift = 16.* tf.random.normal([1],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image['inp1'],tf.transpose(idx3))
return {'inp1': tf.reshape(d,[DIM,DIM,3]), 'inp2': image['inp2']}, label
def cutmix(image, label):
DIM = IMAGE_SIZE[0]
imgs = []; labs = []
for j in range(BATCH_SIZE):
P = tf.cast(tf.random.uniform([], 0, 1)<= cutmix_rate, tf.int32)
 k = tf.cast(tf.random.uniform([], 0, BATCH_SIZE), tf.int32)
 x = tf.cast(tf.random.uniform([], 0, DIM), tf.int32)
 y = tf.cast(tf.random.uniform([], 0, DIM), tf.int32)
 b = tf.random.uniform([], 0, 1)
 WIDTH = tf.cast(DIM * tf.math.sqrt(1-b),tf.int32)* P
ya = tf.math.maximum(0,y-WIDTH//2)
 yb = tf.math.minimum(DIM,y+WIDTH//2)
 xa = tf.math.maximum(0,x-WIDTH//2)
 xb = tf.math.minimum(DIM,x+WIDTH//2)
 one = image['inp1'][j,ya:yb,0:xa,:]
two = image['inp1'][k,ya:yb,xa:xb,:]
three = image['inp1'][j,ya:yb,xb:DIM,:]
middle = tf.concat([one,two,three],axis=1)
 img = tf.concat([image['inp1'][j,0:ya,:,:],middle,image['inp1'][j,yb:DIM,:,:]],axis=0)
 imgs.append(img)
 a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)
 lab1 = label[j,]
lab2 = label[k,]
labs.append(( 1-a)*lab1 + a*lab2)
 image2 = tf.reshape(tf.stack(imgs),(BATCH_SIZE,DIM,DIM,3))
label2 = tf.reshape(tf.stack(labs),(BATCH_SIZE, 1))
return {'inp1': image2, 'inp2': image['inp2']}, label2
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed)
 def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64),
"age_approx": tf.io.FixedLenFeature([], tf.int64),
"sex": tf.io.FixedLenFeature([], tf.int64),
"anatom_site_general_challenge": tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['target'], tf.float32)
 data = {}
data['age_approx'] = tf.cast(example['age_approx'], tf.int32)
 data['sex'] = tf.cast(example['sex'], tf.int32)
 data['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)
 return image, label, data
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
"age_approx": tf.io.FixedLenFeature([], tf.int64),
"sex": tf.io.FixedLenFeature([], tf.int64),
"anatom_site_general_challenge": tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 image_name = example['image_name']
data = {}
data['age_approx'] = tf.cast(example['age_approx'], tf.int32)
 data['sex'] = tf.cast(example['sex'], tf.int32)
 data['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)
 return image, image_name, data
def load_dataset(filenames, labeled = True, ordered = False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO)
 return dataset
def setup_input1(image, label, data):
anatom = [tf.cast(data['anatom_site_general_challenge'][i], dtype = tf.float32)for i in range(7)]
tab_data = [tf.cast(data[tfeat], dtype = tf.float32)for tfeat in ['age_approx', 'sex']]
tabular = tf.stack(tab_data + anatom)
 return {'inp1': image, 'inp2': tabular}, label
def setup_input2(image, image_name, data):
anatom = [tf.cast(data['anatom_site_general_challenge'][i], dtype = tf.float32)for i in range(7)]
tab_data = [tf.cast(data[tfeat], dtype = tf.float32)for tfeat in ['age_approx', 'sex']]
tabular = tf.stack(tab_data + anatom)
 return {'inp1': image, 'inp2': tabular}, image_name
def setup_input3(image, image_name, target, data):
anatom = [tf.cast(data['anatom_site_general_challenge'][i], dtype = tf.float32)for i in range(7)]
tab_data = [tf.cast(data[tfeat], dtype = tf.float32)for tfeat in ['age_approx', 'sex']]
tabular = tf.stack(tab_data + anatom)
 return {'inp1': image, 'inp2': tabular}, image_name, target
def data_augment(data, label):
data['inp1'] = tf.image.random_flip_left_right(data['inp1'])
 data['inp1'] = tf.image.random_flip_up_down(data['inp1'])
 data['inp1'] = tf.image.random_hue(data['inp1'], 0.01)
 data['inp1'] = tf.image.random_saturation(data['inp1'], 0.7, 1.3)
 data['inp1'] = tf.image.random_contrast(data['inp1'], 0.8, 1.2)
 data['inp1'] = tf.image.random_brightness(data['inp1'], 0.1)
 return data, label
def get_training_dataset(filenames, labeled = True, ordered = False):
dataset = load_dataset(filenames, labeled = labeled, ordered = ordered)
 dataset = dataset.map(setup_input1, num_parallel_calls = AUTO)
 dataset = dataset.map(data_augment, num_parallel_calls = AUTO)
 dataset = dataset.map(transform, num_parallel_calls = AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(filenames, labeled = True, ordered = True):
dataset = load_dataset(filenames, labeled = labeled, ordered = ordered)
 dataset = dataset.map(setup_input1, num_parallel_calls = AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(filenames, labeled = False, ordered = True):
dataset = load_dataset(filenames, labeled = labeled, ordered = ordered)
 dataset = dataset.map(setup_input2, num_parallel_calls = AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 def read_tfrecord_full(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"image_name": tf.io.FixedLenFeature([], tf.string),
"target": tf.io.FixedLenFeature([], tf.int64),
"age_approx": tf.io.FixedLenFeature([], tf.int64),
"sex": tf.io.FixedLenFeature([], tf.int64),
"anatom_site_general_challenge": tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 image_name = example['image_name']
target = tf.cast(example['target'], tf.float32)
 data = {}
data['age_approx'] = tf.cast(example['age_approx'], tf.int32)
 data['sex'] = tf.cast(example['sex'], tf.int32)
 data['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)
 return image, image_name, target, data
def load_dataset_full(filenames):
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.map(read_tfrecord_full, num_parallel_calls = AUTO)
 return dataset
def get_data_full(filenames):
dataset = load_dataset_full(filenames)
 dataset = dataset.map(setup_input3, num_parallel_calls = AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
NUM_TRAINING_IMAGES = int(count_data_items(TRAINING_FILENAMES)* 0.8)
 NUM_VALIDATION_IMAGES = int(count_data_items(TRAINING_FILENAMES)* 0.2)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE
print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))<compute_train_metric>
days_prog = 30
Matrix_confirmed,Matrix_Death ={},{}
for model_name in model_Confirmed:
new_data_list = new_data[new_data.days_mart == days_x]
labels = new_data_list.countries
predictions_confirmed = []
predictions_Death = []
new_data_list = new_data_list.drop(['Province_State', 'ForecastId','confirmed','countries','deaths'], axis='columns')
 for _ in range(days_prog):
prediction_confirmed = model_Confirmed[model_name].predict(new_data_list)
 prediction_Death = model_Death[model_name].predict(new_data_list)
 prediction_confirmed = prediction_confirmed*(0.95**(new_data_list.days_after_Quarantine.values/10 - 1.4))
prediction_Death = prediction_Death*(0.95**(new_data_list.days_after_Quarantine.values/10 - 1.4))
prediction_confirmed = prediction_confirmed + new_data_list.pred_conf.values
prediction_Death = prediction_Death + new_data_list.pred_deaths.values

 new_data_list["delta_conf2"] = new_data_list["delta_conf1"]
new_data_list["delta_conf1"] = prediction_confirmed - new_data_list["pred_conf"]
new_data_list["pred_conf"] = prediction_confirmed
new_data_list["delta_deaths2"] = new_data_list["delta_deaths1"]
new_data_list["delta_deaths1"] = prediction_Death - new_data_list["pred_deaths"]
new_data_list["pred_deaths"] = prediction_Death
prediction_Death = 10**prediction_Death - 1
prediction_confirmed = 10**prediction_confirmed - 1
new_data_list[["days_mart", "days"]] += 1
predictions_Death.append(np.round(prediction_Death))
predictions_confirmed.append(np.round(prediction_confirmed))
data_list = pd.date_range('2020-04-'+str(days_x-30), periods = days_prog, freq ='d')
 data_list = data_list.strftime('%
Matrix_confirmed[model_name] = pd.DataFrame(predictions_confirmed,columns = labels,index = data_list)
 Matrix_Death[model_name] = pd.DataFrame(predictions_Death,columns = labels,index = data_list)
 Matrix_confirmed["Среднее"] =(Matrix_confirmed["Xgboost"] + Matrix_confirmed["RandomForest"] + Matrix_confirmed["Adaboost"])//3
Matrix_Death["Среднее"] =(Matrix_Death["Xgboost"] + Matrix_Death["RandomForest"] + Matrix_Death["Adaboost"])//3
def binary_focal_loss(gamma=2., alpha=.25):

 def binary_focal_loss_fixed(y_true, y_pred):

 pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
epsilon = K.epsilon()
pt_1 = K.clip(pt_1, epsilon, 1.- epsilon)
 pt_0 = K.clip(pt_0, epsilon, 1.- epsilon)
 return -K.sum(alpha * K.pow(1.- pt_1, gamma)* K.log(pt_1)) \
-K.sum(( 1 - alpha)* K.pow(pt_0, gamma)* K.log(1.- pt_0))
return binary_focal_loss_fixed
def get_model() :
with strategy.scope() :
inp1 = tf.keras.layers.Input(shape =(*IMAGE_SIZE, 3), name = 'inp1')
 inp2 = tf.keras.layers.Input(shape =(9), name = 'inp2')
 efnetb3 = efn.EfficientNetB3(weights = 'imagenet', include_top = False)
 x = efnetb3(inp1)
 x = tf.keras.layers.GlobalAveragePooling2D()(x)
 x1 = tf.keras.layers.Dense(50 )(inp2)
 x1 = tf.keras.layers.BatchNormalization()(x1)
 x1 = tf.keras.layers.Activation('relu' )(x1)
 concat = tf.keras.layers.concatenate([x, x1])
 concat = tf.keras.layers.Dense(512, activation = 'relu' )(concat)
 concat = tf.keras.layers.BatchNormalization()(concat)
 concat = tf.keras.layers.Dropout(0.2 )(concat)
 concat = tf.keras.layers.Dense(182, activation = 'relu' )(concat)
 concat = tf.keras.layers.BatchNormalization()(concat)
 concat = tf.keras.layers.Dropout(0.2 )(concat)
 output = tf.keras.layers.Dense(1, activation = 'sigmoid' )(concat)
 model = tf.keras.models.Model(inputs = [inp1, inp2], outputs = [output])
 opt = tf.keras.optimizers.Adam(learning_rate = LR)
 model.compile(
 optimizer = opt,
loss = [binary_focal_loss(gamma = 2.0, alpha = 0.80)],
metrics = [tf.keras.metrics.BinaryAccuracy() , tf.keras.metrics.AUC() ] 
)
 return model
def train_and_predict(SUB, folds = 5):
models = []
oof_image_name = []
oof_target = []
oof_prediction = []
seed_everything(SEED)
 kfold = KFold(folds, shuffle = True, random_state = SEED)
 for fold,(trn_ind, val_ind)in enumerate(kfold.split(TRAINING_FILENAMES)) :
print('
')
 print('-'*50)
 print(f'Training fold {fold + 1}')
 train_dataset = get_training_dataset([TRAINING_FILENAMES[x] for x in trn_ind], labeled = True, ordered = False)
 val_dataset = get_validation_dataset([TRAINING_FILENAMES[x] for x in val_ind], labeled = True, ordered = True)
 K.clear_session()
model = get_model()
early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_auc', mode = 'max', patience = 5,
verbose = 1, min_delta = 0.0001, restore_best_weights = True)
 cb_lr_schedule = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'val_auc', factor = 0.4, patience = 2, verbose = 1, min_delta = 0.0001, mode = 'max')
 history = model.fit(train_dataset,
steps_per_epoch = STEPS_PER_EPOCH,
epochs = EPOCHS,
callbacks = [early_stopping, cb_lr_schedule],
validation_data = val_dataset,
verbose = 2)
 models.append(model)
 number_of_files = count_data_items([TRAINING_FILENAMES[x] for x in val_ind])
 dataset = get_data_full([TRAINING_FILENAMES[x] for x in val_ind])
 image_name = dataset.map(lambda image, image_name, target: image_name ).unbatch()
image_name = next(iter(image_name.batch(number_of_files)) ).numpy().astype('U')
 target = dataset.map(lambda image, image_name, target: target ).unbatch()
target = next(iter(target.batch(number_of_files)) ).numpy()
image = dataset.map(lambda image, image_name, target: image)
 probabilities = model.predict(image)
 oof_image_name.extend(list(image_name))
oof_target.extend(list(target))
oof_prediction.extend(list(np.concatenate(probabilities)))
 print('
')
 print('-'*50)
 oof_df = pd.DataFrame({'image_name': oof_image_name, 'target': oof_target, 'predictions': oof_prediction})
 oof_df.to_csv('EfficientNetB3_384.csv', index = False)
 test_ds = get_test_dataset(TEST_FILENAMES, labeled = False, ordered = True)
 test_images_ds = test_ds.map(lambda image, image_name: image)
 print('Computing predictions...')
 probabilities = np.average([np.concatenate(models[i].predict(test_images_ds)) for i in range(folds)], axis = 0)
 print('Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, image_name: image_name ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 pred_df = pd.DataFrame({'image_name': test_ids, 'target': probabilities})
 SUB.drop('target', inplace = True, axis = 1)
 SUB = SUB.merge(pred_df, on = 'image_name')
 SUB.to_csv('sub_EfficientNetB3_384.csv', index = False)
 return oof_target, oof_prediction
oof_target, oof_prediction = train_and_predict(SUB )<compute_test_metric>
ans = Matrix_confirmed["Adaboost"].stack().reset_index()
ans_Death = Matrix_Death["Adaboost"].stack().reset_index()
ans["prediction_deaths"] = ans_Death[0]
ans["Province_State"] = new_data.groupby(["countries", "Province_State"] ).max().reset_index() ["Province_State"].tolist() *30
ans.columns = ['Date','Country_Region','ConfirmedCases', 'Fatalities',"Province_State"]
ans['Date'] = '2020-'+ans['Date']
train2 = train[(train.Date >= "2020-04-02")]
train2['Province_State'] = train2['Province_State'].fillna("zzz")
 train2.drop(['Id'], axis='columns',inplace=True)
 ans = pd.concat([train2, ans],ignore_index=True)
 ans['ConfirmedCases'] = ans['ConfirmedCases']
ans['Fatalities'] = ans['Fatalities']
ans = ans.sort_values(by=['Country_Region','Province_State','Date'])
 ans['ForecastId'] = np.arange(1,len(ans)+1)
 ans = ans[['ForecastId','ConfirmedCases','Fatalities']]
ans.to_csv('submission.csv', index=False )