Kaggle competition: COVID19 Global Forecasting (Week 4)
sub.target =(public_sub_9619.target * 0.35 +
public_sub_9606.target *0.35 +
public_sub_tabular.target * 0.30 )<save_to_csv>
df=pd.read_csv("/kaggle/input/covid19-global-forecasting-week-4/train.csv")
 test=pd.read_csv("/kaggle/input/covid19-global-forecasting-week-4/test.csv" )
sub.head()
sub.to_csv('submission.csv', index = False )<import_modules>
df["Province_State"].fillna("state", inplace = True)
 df["Country_Region"] = [country_name.replace("'","")for country_name in df["Country_Region"]]
print(df.shape,"
",df.head() )
import numpy as np
import pandas as pd<load_from_csv>
X_y=shuffle(new_data)
 y_cases=X_y['target_infection']
y_fatal=X_y['target_fatal']
X=X_y.drop(['target_infection','target_fatal'],axis=1)
 X_train_cases, X_test_cases, y_train_cases, y_test_cases = train_test_split(X, y_cases, test_size=0.33)
 X_train_fatal, X_test_fatal, y_train_fatal, y_test_fatal = train_test_split(X, y_fatal, test_size=0.33)
 print("Shape of infection train dataset:",(X_train_cases.shape,y_train_cases.shape))
print("Shape of infection test dataset:",(X_test_cases.shape,y_test_cases.shape))
print("Shape of fatal train dataset:",(X_train_fatal.shape,y_train_fatal.shape))
print("Shape of fatal test dataset:",(X_test_fatal.shape,y_test_fatal.shape))
!wget -q -O other1.csv https://www.kaggleusercontent.com/kf/36961390/eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0.. k0-Yl9R-H0lC86Nth0mIMQ.HiygAXAzwROTh0HZTHHGexh-f2l5eaPH1wFhHwsZxcrhqsddvY1Vrvy8xnKkzleMoYKR2DFzwKr3wpX3JkzC74BsxlPSpA5WeSAJkUitCktmGkcdB-EOiusI6rEBYMJUaXgx52hzF-dcVTPgRlj2HgnLCixBfs78h7mgFokLo1kZjgXjWlDHXcRao9DGKKHvp2EZxkGJ3-crDI_7kLxFfpDDlUjxmtLSZTk6yNxnExsj-nljUOan0tu7dHihhrrABxR7ZeT55MKqhkqew8slishHNIfnUvZug-n50E_yf0fjh8ydo7LUVYOVT4oY3WllNNhDc1PIEWXau9MPJITQZ6zW_KxYt9qRPkfCrXAr8UXtffECQ7Ut0zUcn7CUo1qen4fBLVhHnixAPnD6rFlv_23qvSgWRXbHQSVcjlSpC2tKWfVTFswN2zRREiahEiVbxWRH3vDrHk0uJeWSmHPUqS7cYT71h0bZhtmuOWd8T5hhk1hC4tECcSuZvxG80woxHsitF0YiQjDFQTTL7T3vUBSQYzLeAnTs4N5iZNVpyaIBVJJDCQ76yZ1lvDK9126jMASTxwnZVjb5mUAuYTpkxpS10tg-V8pzIsyoqitRAgWYiiMZ-AWdofBltCqZbW6ZCAshq9jSwr9dhjoDt1H5PD6m8A4Sko8i0oGH4YqoGMk.XzhuO7hsNjxzjnivzC2boA/sub_EfficientNetB3_384.csv<load_from_csv>
reg_case=ElasticNet(random_state=42,l1_ratio=0.1,max_iter=2200)
 params = [{'alpha': [10**-4,10**-3, 10**-2,10**-1, 10**0,10**1, 10**2,10**3,10**4]}]
clf = RandomizedSearchCV(reg_case, params, cv=4, scoring='neg_root_mean_squared_error',return_train_score=True)
 search=clf.fit(X_train_cases, y_train_cases)
 results = pd.DataFrame.from_dict(clf.cv_results_)
a = pd.read_csv('other1.csv')
 b = pd.read_csv('.. /input/stack-models/submission.csv')
 c = pd.read_csv('.. /input/melanoma-efficientnet-b6-tpu-tta/submission.csv' )<save_to_csv>
best_alpha=1000
best_itr=2400
final_reg_case=ElasticNet(random_state=42,alpha=best_alpha,l1_ratio=0.1,max_iter=best_itr)
 final_reg_case.fit(X_train_cases,y_train_cases )
pd.DataFrame({'image_name': a.image_name, 'target':(a.target + c.target*0.8)/2} ).to_csv('submission.csv', index=False )<define_variables>
pred=final_reg_case.predict(X_test_cases)
 print("The RMSE value",(mean_squared_error(y_test_cases,pred)) **0.5 )
DEVICE = "TPU"
CFG = dict(
 net_count = 7,
batch_size = 32,
read_size = 512,
crop_size = 512,
net_size = 512,
LR_START = 0.000005,
LR_MAX = 0.000020,
LR_MIN = 0.000001,
LR_RAMPUP_EPOCHS = 7,
LR_SUSTAIN_EPOCHS = 0,
LR_EXP_DECAY = 0.8,
epochs = 25,
rot = 180.0,
shr = 2.0,
hzoom = 8.0,
wzoom = 8.0,
hshift = 8.0,
wshift = 8.0,
optimizer = 'adam',
label_smooth_fac = 0.05,
tta_steps = 25
)<install_modules>
reg_fatal=ElasticNet(random_state=42,l1_ratio=0.1,max_iter=3500)
 params = [{'alpha': [10**-4,10**-3, 10**-2,10**-1, 10**0,10**1, 10**2,10**3,10**4]}]
clf = RandomizedSearchCV(reg_fatal, params, cv=4, scoring='neg_root_mean_squared_error',return_train_score=True)
 search=clf.fit(X_train_fatal, y_train_fatal)
 results = pd.DataFrame.from_dict(clf.cv_results_)
!pip install -q efficientnet<import_modules>
best_alpha=100
best_iter=3500
final_reg_fatal = ElasticNet(random_state=42,alpha=best_alpha,l1_ratio=0.1,max_iter=best_iter)
 final_reg_fatal.fit(X_train_fatal, y_train_fatal )
random.seed(a=42)
<load_from_csv>
pred=final_reg_fatal.predict(X_test_fatal)
 print("The RMSE value",(mean_squared_error(y_test_fatal,pred)) **0.5 )
BASEPATH = ".. /input/siim-isic-melanoma-classification"
df_train = pd.read_csv(os.path.join(BASEPATH, 'train.csv'))
df_test = pd.read_csv(os.path.join(BASEPATH, 'test.csv'))
df_sub = pd.read_csv(os.path.join(BASEPATH, 'sample_submission.csv'))
GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-512x512')
 GCS_PATH_2019 = KaggleDatasets().get_gcs_path('isic2019-512x512')
 files_train_2019 = np.array(tf.io.gfile.glob(GCS_PATH_2019 + '/train*.tfrec'))
files_train_2020 = np.array(tf.io.gfile.glob(GCS_PATH + '/train*.tfrec'))
files_train = np.sort(np.concatenate(( files_train_2020, files_train_2019)))
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')) )<choose_model_class>
featured=pd.DataFrame(data)
 X_y_f=shuffle(featured)
 y_cases_f=X_y_f['target_infection']
y_fatal_f=X_y_f['target_fatal']
X_f=X_y_f.drop(['target_infection','target_fatal'],axis=1)
 X_train_cases_f, X_test_cases_f, y_train_cases_f, y_test_cases_f = train_test_split(X_f, y_cases_f, test_size=0.33)
 X_train_fatal_f, X_test_fatal_f, y_train_fatal_f, y_test_fatal_f = train_test_split(X_f, y_fatal_f, test_size=0.33)
 print("Shape of featurized infection train dataset:",(X_train_cases_f.shape,y_train_cases_f.shape))
print("Shape of featurized infection test dataset:",(X_test_cases_f.shape,y_test_cases_f.shape))
print("Shape of featurized fatal train dataset:",(X_train_fatal_f.shape,y_train_fatal_f.shape))
print("Shape of featurized fatal test dataset:",(X_test_fatal_f.shape,y_test_fatal_f.shape))
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )<normalization>
reg_case_f=ElasticNet(random_state=42,l1_ratio=0.1,max_iter=2200)
 params = [{'alpha': [10**-4,10**-3, 10**-2,10**-1, 10**0,10**1, 10**2,10**3,10**4]}]
clf_f= RandomizedSearchCV(reg_case_f, params, cv=4, scoring='neg_root_mean_squared_error',return_train_score=True)
 search_f=clf_f.fit(X_train_cases_f, y_train_cases_f)
 results_f = pd.DataFrame.from_dict(clf_f.cv_results_ )
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, cfg):
DIM = cfg["read_size"]
XDIM = DIM%2
rot = cfg['rot'] * tf.random.normal([1], dtype='float32')
 shr = cfg['shr'] * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ cfg['hzoom']
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ cfg['wzoom']
h_shift = cfg['hshift'] * tf.random.normal([1], dtype='float32')
 w_shift = cfg['wshift'] * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )<prepare_x_and_y>
best_alpha=10000
best_itr=4200
final_reg_case_f=ElasticNet(random_state=42,alpha=best_alpha,l1_ratio=0.1,max_iter=best_itr)
 final_reg_case_f.fit(X_train_cases_f,y_train_cases_f )
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'diagnosis' : tf.io.FixedLenFeature([], tf.int64),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return(example['image'],(example['sex'], example['age_approx'], example['anatom_site_general_challenge']), example['target'])
 def read_unlabeled_tfrecord(example, return_image_name):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return(example['image'],(example['sex'], example['age_approx'], example['anatom_site_general_challenge']),(example['image_name'] if return_image_name else 0))
def prepare_data(data, cfg=None, augment=True):
img = data[0]
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.image.resize(img, [cfg['read_size'], cfg['read_size']])
 img = tf.cast(img, tf.float32)/ 255.0
if augment:
img = transform(img, cfg)
 img = tf.image.random_crop(img, [cfg['crop_size'], cfg['crop_size'], 3])
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_hue(img, 0.01)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 else:
img = tf.image.central_crop(img, cfg['crop_size'] / cfg['read_size'])
 img = tf.image.resize(img, [cfg['net_size'], cfg['net_size']])
 img = tf.reshape(img, [cfg['net_size'], cfg['net_size'], 3])
 sex_oh = tf.one_hot(data[1][0], 2)
 age_aprox = tf.dtypes.cast(tf.reshape(data[1][1], [1]), tf.float32)
 anatom_site_general_challenge = tf.one_hot(data[1][2], 7)
 dense = tf.concat([sex_oh, age_aprox, anatom_site_general_challenge], axis=0)
 return(img, dense)
 def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )<categorify>
pred_f=final_reg_case_f.predict(X_test_cases_f)
 print("RMSE is:",(mean_squared_error(y_test_cases_f,pred_f)) **0.5 )
def get_dataset(files, cfg, augment = False, shuffle = False, repeat = False,
labeled=True, return_image_names=True):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTO)
 ds = ds.map(lambda img, dense, imgname_or_label:(prepare_data(( img, dense), augment=augment, cfg=cfg),
imgname_or_label),
num_parallel_calls=AUTO)
 ds = ds.batch(cfg['batch_size'] * REPLICAS)
 ds = ds.prefetch(AUTO)
 return ds<data_type_conversions>
reg_fatal_f=ElasticNet(random_state=42,alpha=best_alpha,l1_ratio=0.1,max_iter=2200)
 params = [{'alpha': [10**-4,10**-3, 10**-2,10**-1, 10**0,10**1, 10**2,10**3,10**4]}]
clf_f= RandomizedSearchCV(reg_fatal_f, params, cv=4, scoring='neg_root_mean_squared_error',return_train_score=True)
 search_f=clf_f.fit(X_train_fatal_f, y_train_fatal_f)
 results_f = pd.DataFrame.from_dict(clf_f.cv_results_ )
def show_dataset(thumb_size, cols, rows, ds):
mosaic = PIL.Image.new(mode='RGB', size=(thumb_size*cols +(cols-1),
thumb_size*rows +(rows-1)))
 for idx, data in enumerate(iter(ds)) :
img, target_or_imgid = data[0][0], data[1]
ix = idx % cols
iy = idx // cols
img = np.clip(img.numpy() * 255, 0, 255 ).astype(np.uint8)
 img = PIL.Image.fromarray(img)
 img = img.resize(( thumb_size, thumb_size), resample=PIL.Image.BILINEAR)
 mosaic.paste(img,(ix*thumb_size + ix,
iy*thumb_size + iy))
display(mosaic)
 ds = get_dataset(files_train, CFG ).unbatch().take(12*5)
 show_dataset(64, 12, 5, ds )<count_unique_values>
best_alpha=100
best_itr=2400
final_reg_fatal_f=ElasticNet(random_state=42,alpha=best_alpha,l1_ratio=0.1,max_iter=best_itr)
 final_reg_fatal_f.fit(X_train_fatal_f,y_train_fatal_f )
len(df_train.diagnosis.unique() )<create_dataframe>
pred_f=final_reg_fatal_f.predict(X_test_fatal_f)
 print("RMSE is:",(mean_squared_error(y_test_fatal_f,pred_f)) **0.5 )
ds = tf.data.TFRecordDataset(files_train, num_parallel_reads=AUTO)
 ds = ds.take(1 ).cache().repeat()
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 ds = ds.map(lambda img, dense, target:(prepare_data(( img, dense), cfg=CFG, augment=True), target),
num_parallel_calls=AUTO)
 ds = ds.take(12*5)
 ds = ds.prefetch(AUTO)
 show_dataset(64, 12, 5, ds )<create_dataframe>
test["Province_State"].fillna("state", inplace = True)
 test["Country_Region"] = [country_name.replace("'","")for country_name in test["Country_Region"]]
ds = get_dataset(files_test, CFG, augment=True, repeat=True,
labeled=False, return_image_names=False ).unbatch().take(12*5)
 show_dataset(64, 12, 5, ds )<choose_model_class>
test['ConfirmedCases']=list(map(int,predicted_case))
test['Fatalities']=list(map(int,predicted_fatal))
def get_lr_callback(cfg):
lr_start = cfg['LR_START']
lr_max = cfg['LR_MAX'] * strategy.num_replicas_in_sync
lr_min = cfg['LR_MIN']
lr_ramp_ep = cfg['LR_RAMPUP_EPOCHS']
lr_sus_ep = cfg['LR_SUSTAIN_EPOCHS']
lr_decay = cfg['LR_EXP_DECAY']
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback<choose_model_class>
submission=test[['ForecastId','ConfirmedCases','Fatalities']]
submission=shuffle(submission)
 submission.to_csv("submission.csv",index=False )