Kaggle competition: COVID19 Global Forecasting (Week 4)
%%skip SUBMIT or not train_nn
model_path="best_nn_model.h5"
callbacks_list = [
keras.callbacks.ModelCheckpoint(model_path,
verbose=0,
monitor='auc',
save_best_only=True,
mode='max'),
]
batch_size = 2**14
val_data = None if X_test is None else(X_test,y_test )<choose_model_class>
PATH_WEEK4 = '/kaggle/input/covid19-global-forecasting-week-4'
df_Train = pd.read_csv(f'{PATH_WEEK4}/train.csv', parse_dates=["Date"], engine='python')
 df_Test = pd.read_csv(f'{PATH_WEEK4}/test.csv', parse_dates=["Date"], engine='python')
%%skip SUBMIT or not train_nn
layers = [keras.layers.BatchNormalization(input_shape=(len(features),)) ,]
for u in range(5):
layers.extend([
keras.layers.BatchNormalization(input_shape=(len(features),)) ,
keras.layers.Dense(256),
keras.layers.BatchNormalization() ,
keras.layers.Activation('relu'),
])
 model = keras.Sequential([
*layers,
keras.layers.Dense(1,activation='sigmoid'),
])
 if os.path.exists("nn_model.h5"):
print('Loading...')
 model = keras.models.load_model("nn_model.h5")
 model.compile(loss=keras.losses.binary_crossentropy,
optimizer=keras.optimizers.Adam(0.01),
metrics=[
'acc',
keras.metrics.AUC(name='auc'),
] 
)
 model.summary()<train_model>
def fillState(state, country):
if state == EMPTY_VAL: return country
return state
%%skip SUBMIT or not train_nn
history = model.fit(X,y,
batch_size=batch_size,
validation_data=val_data,
epochs=100,
callbacks=callbacks_list, 
)
 model.save("nn_model.h5" )<set_options>
df_Train.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_Test.rename(columns={'Country_Region':'Country'}, inplace=True)
 EMPTY_VAL = "EMPTY_VAL"
df_Train.rename(columns={'Province_State':'State'}, inplace=True)
 df_Train['State'].fillna(EMPTY_VAL, inplace=True)
 df_Train['State'] = df_Train.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 df_Test.rename(columns={'Province_State':'State'}, inplace=True)
 df_Test['State'].fillna(EMPTY_VAL, inplace=True)
 df_Test['State'] = df_Test.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1 )
%%skip not SUBMIT
lock = mp.Lock()<define_variables>
MIN_TEST_DATE = df_Test.Date.min()
df_train = df_Train.loc[df_Train.Date < MIN_TEST_DATE, :]
y1_Train = df_train.iloc[:, -2]
y2_Train = df_train.iloc[:, -1]
use_lgb = True
use_xgb = False
use_nn = False<load_pretrained>
def extractDate(df, colName = 'Date'):

 assert colName in df.columns
df = df.assign(
 week = df.loc[:, colName].dt.week,
month = df.loc[:, colName].dt.month,
day = df.loc[:, colName].dt.day,
dayofweek = df.loc[:, colName].dt.dayofweek,
dayofyear = df.loc[:, colName].dt.dayofyear, 
)
 return df
%%skip not(use_lgb and SUBMIT)
 model = dill.load(open('lgb_model.dill','rb'))
model<load_pretrained>
def createNewDataset(df):

 df_New = df.copy()
df_New.loc[:, 'Date_Int'] =(df_New.loc[:, 'Date'].dt.strftime("%m%d")).astype('int16')
 df_New.drop(columns=['Date'], axis=1, inplace=True)
 return df_New
%%skip not(use_xgb and SUBMIT)
 model = dill.load(open('xgb_model.dill','rb'))
model<load_pretrained>
X_Train = createNewDataset(df_train)
 X_Test = createNewDataset(df_Test )
%%skip not(use_nn and SUBMIT)
 model = keras.models.load_model("best_nn_model.h5" )<load_pretrained>
days_change = [1, 2, 3, 5, 7, 10]
def getChangeGrowth(df):
newCCols = []
newGCols = []
for day in days_change:
newCCol = f'C_{day}'
df['C'+newCCol] = df['LConfirmedCases'] - df[f'CD_{day}']
df['F'+newCCol] = df['LFatalities'] - df[f'FD_{day}']
newCCols.append(newCCol)
 newGCol = f'G_{day}'
df['C'+newGCol] = df['C'+newCCol] / df[f'CD_{day}']
df['F'+newGCol] = df['F'+newCCol] / df[f'FD_{day}']
newGCols.append(newGCol)
 df.fillna(0, inplace=True)
 return df
%%skip not SUBMIT
metadata_info = dill.load(open('/kaggle/input/riiid-metadata-to-sqlite/metadata_info.dill','rb'))
features = dill.load(open('features.dill','rb'))
ignore_columns = dill.load(open('ignore_columns.dill','rb'))
content_df = pd.read_feather('.. /input/riiid-preprocess-and-balance-the-dataset/content.feather')
 print('Features size: ',len(features))<feature_engineering>
windows = [1, 2, 3, 5, 7]
def getMA(df):
newCMACols = []
newGMACols = []
for window in windows:
for day in days_change:
newCMACol = f'CMA_{day}_{window}'
df['C'+newCMACol] = df[f'CC_{day}'].rolling(window ).mean()
df['F'+newCMACol] = df[f'FC_{day}'].rolling(window ).mean()
newCMACols.append(newCMACol)
 newGMACol = f'GMA_{day}_{window}'
df['C'+newGMACol] = df[f'CG_{day}'].rolling(window ).mean()
df['F'+newGMACol] = df[f'FG_{day}'].rolling(window ).mean()
newGMACols.append(newGMACol)
 df.fillna(0, inplace=True)
 return df
%%skip not SUBMIT
for key in metadata_info:
name = f"db.{'_'.join(key)}.sqlite"
engine = create_engine(f'sqlite:///{name}?check_same_thread=False', echo=False)
 sqlite_connection = engine.connect()
cols = metadata_info[key][-1]
cols = [column(col)for col in cols]+[column('index'),]
tab = table('_'.join(key), *cols)
 metadata_info[key][0] =(sqlite_connection,tab )<feature_engineering>
cases = [1, 50, 100, 500, 1000, 5000, 35000, 75000, 100000]
def getCDSC(df):
newCDSCCols = []
for case in cases:
newDSCCol = f'{case}_CDSC'
df.loc[df.CD_1 == 0, newDSCCol] = 0
df.loc[df.CD_1 >= case, newDSCCol] = df[df.CD_1 >= case].groupby(['Country', 'State'] ).cumcount()
newCDSCCols.append(newDSCCol)
 df.fillna(0, inplace=True)
 return df
def build_index(x):
return '_'.join(map(lambda x:str(int(x)) ,x)) if type(x)not in [int,float] else str(int(x))
def getValues(args):
key,df,cache= args
df = df[list(key)]
index = df.index
df = df[list(key)]
db,tab = metadata_info[key][0]
cols = metadata_info[key][-1]
ids = df.apply(build_index,axis=1 ).values
query = select([tab.c['index'],] + [tab.c[col] for col in cols] ).where(tab.c.index.in_(np.unique(ids)))
 result = list(db.execute(query))
res_df = pd.DataFrame(result,columns=['tmp_key']+cols)
 res_df.set_index('tmp_key',inplace=True)
 df['tmp_key'] = ids
df = df.merge(res_df,how='left',on='tmp_key' ).fillna(0)
 cache[key] = df.groupby([*key,] ).tail(1 ).set_index([*key,])
 df = df.drop(columns=[*key,'tmp_key'])
 df.index = index
return df
def merge_metadata(df):
cache = {}
workers= []
queue = Queue()
for key in metadata_info:
worker = Worker(getValues,(key,df,cache),queue)
 worker.start()
workers.append(worker)
 frames = [queue.get() for _ in workers]
frames.append(df)
 for worker in workers:
worker.join()
return pd.concat(frames,axis=1),cache
def update_metadata(args):
df,key = args
if not df.shape[0]:
return
key =(key,)if type(key)== str else tuple(key)
 if key not in metadata_info:
return 
(db,tab),cols = metadata_info[key]
cols = [c for c in cols if c not in ignore_columns]
if all(k in df.index.names for k in key):
df = df.loc[:,[c for c in cols if c not in key]]
else:
df = df.set_index(list(key)).loc[:,cols]
db_index = list(map(build_index,df.index))
df.index = db_index
query = delete(tab ).where(tab.c.index.in_(db_index))
db.execute(query)
 df.to_sql('_'.join(key), db, if_exists='append')
 def add_previous_state(df,key,cache):
key =(key,)if type(key)== str else tuple(key)
 if not all(k in df.index.names for k in key):
df.set_index(list(key),inplace=True)
 index = df.index
cols = [c for c in df.columns if c not in ignore_columns]
if not cols:
return df
return(df+cache[key])[cols]<groupby>
deaths = [1, 50, 100, 500, 1000, 5000, 35000]
def getFDSC(df):
newFDSCCols = []
for death in deaths:
newDSCCol = f'{death}_FDSC'
df.loc[df.FD_1 == 0, newDSCCol] = 0
df.loc[df.FD_1 >= death, newDSCCol] = df[df.FD_1 >= death].groupby(['Country', 'State'] ).cumcount()
newFDSCCols.append(newDSCCol)
 df.fillna(0, inplace=True)
 return df
def user_roll_features(df,cols):
metadata = {}
for col in cols:
c1 = f'{col}_roll_count'
c2 = f'{col}_roll_sum'
df_tmp = df[[col,'user_id','answered_correctly',]]
grp = df_tmp.groupby([col,'user_id'])
 df_tmp[c1] = grp.answered_correctly.cumcount() +1
df_tmp[c2] = grp.answered_correctly.cumsum()
wincols = []
for win in [50,10]:
sum_col = f'{col}_win{win}_sum'
count_col = f'{col}_win{win}_count'
df_tmp[sum_col] = df_tmp[c2]
df_tmp[count_col] = df_tmp[c1]
wincols.append(sum_col)
 wincols.append(count_col)
 grp = df_tmp.groupby(['user_id',col] ).tail(1)
 metadata[('user_id',col)] = grp.set_index(['user_id',col])[[c1,c2,*wincols]]
return metadata<define_variables>
df = pd.concat([df_Train, df_Test[df_Test.Date > df_Train.Date.max() ]], axis=0, sort=False, ignore_index=True )
skip_test = True<load_from_csv>
df['LConfirmedCases'] = np.log1p(df['ConfirmedCases'])
 df['LFatalities'] = np.log1p(df['Fatalities'] )
%%skip skip_test
val_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',
header=0,
skiprows=range(1,1000000*5),
nrows=500000)
 val_df['row_id'] = range(val_df.shape[0])
 val_df = val_df[['row_id', 'timestamp', 'user_id', 'content_id', 'content_type_id',
'task_container_id', 'prior_question_elapsed_time',
'prior_question_had_explanation',
'answered_correctly','user_answer']]<categorify>
df.loc[(df.Date >= df_Test.Date.min())&(df.Date <= df_Train.Date.max()), 'ForecastId'] = df_Test.loc[(df_Test.Date >= df_Test.Date.min())&(df_Test.Date <= df_Train.Date.max()), 'ForecastId'].values
%%skip skip_test
prev_test_df = None
prev_cache = None
auc_sum = 0
acc_sum = 0
df = val_df
count = 0
while df.shape[0]:
count+=1
selector = df.task_container_id == df.task_container_id.min()
test_df = df.loc[selector]
df = df.loc[~selector]
print(f'{test_df.shape[0]}/{df.shape[0]}')
 test_df = test_df.merge(content_df,how='left',on=['content_id',])
 test_df.prior_question_had_explanation = test_df.prior_question_had_explanation.fillna(False ).astype(np.int8)
 test_state = test_df.loc[test_df['content_type_id'] == 0]
test_state.replace([np.inf, -np.inf], 999999999,inplace=True)
 test_state,cache = merge_metadata(test_state)
 test_state['prior_duration'] = test_state.timestamp-test_state.prev_timestamp
test_state = add_mean(test_state)
 test_state['answered_correctly'] = model.predict(test_state[features])
 y = test_df[test_df['content_type_id'] == 0]['answered_correctly']
y_pred = test_state['answered_correctly']
auc = roc_auc_score(y, y_pred)
 acc = accuracy_score(y, y_pred>0.5)
 auc_sum += auc
acc_sum += acc
print(f"AUC:{round(auc,4)} | Accuracy: {round(acc,4)} | AUC MEAN: {round(auc_sum/count,4)} | ACC MEAN: {round(acc_sum/count,4)}")
 if prev_test_df is not None:
workers = []
prev_test_df = prev_test_df[prev_test_df.content_type_id == 0]
user_df = prev_test_df[['user_id','answered_correctly']]
grp = user_df.groupby(['user_id'])
 user_df['user_id_roll_sum'] = grp['answered_correctly'].cumsum()
user_df['user_id_roll_count'] = grp['answered_correctly'].cumcount() +1
user_df = user_df.groupby(['user_id',] ).tail(1)
 user_df.set_index('user_id',inplace=True)
 del user_df['answered_correctly']
win_cols = []
for win in [50,10]:
sum_col = f'user_id_win{win}_sum'
count_col = f'user_id_win{win}_count'
user_df[sum_col] = user_df.user_id_roll_sum
user_df[count_col] = user_df.user_id_roll_count
win_cols.append(sum_col)
 win_cols.append(count_col)
 win_current_state = user_df[win_cols]
user_df = add_previous_state(user_df,('user_id',),prev_cache)
 for win in [50,10]:
sum_col = f'user_id_win{win}_sum'
count_col = f'user_id_win{win}_count'
selector =(user_df[count_col]>win ).values
user_df.loc[selector,[sum_col,count_col]] = win_current_state.loc[selector,[sum_col,count_col]]
tmp_df = prev_test_df[['user_answer','answered_correctly','user_id','timestamp']].groupby('user_id' ).tail(1)
 tmp_df.set_index('user_id',inplace=True)
 tmp_df.columns = [f'prev_{col}' for col in tmp_df.columns]
user_df = user_df.merge(tmp_df,how='left',on=['user_id',])
 worker = Worker(update_metadata,(user_df,['user_id'],))
worker.start()
workers.append(worker)
 cols = ['content_id','bundle_id',]
metadata = user_roll_features(prev_test_df,cols)
 for key in metadata:
col = key[-1]
tmp_df = metadata[key]
win_current_state = tmp_df
tmp_df = add_previous_state(tmp_df,key,prev_cache)
 for win in [50,10]:
sum_col = f'{col}_win{win}_sum'
count_col = f'{col}_win{win}_count'
selector =(tmp_df[count_col]>win ).values
tmp_df.loc[selector,[sum_col,count_col]] = win_current_state.loc[selector,[sum_col,count_col]]
if key ==('user_id','content_id'):
tmp = prev_test_df[['content_id','user_id','answered_correctly',]]
tmp.rename(columns={'answered_correctly':'prev_content_answered_correctly'},inplace=True)
 tmp = tmp.groupby(['content_id','user_id',] ).tail(1)
 tmp.set_index(['user_id','content_id'],inplace=True)
 tmp_df = tmp_df.merge(tmp,how='left',on=['user_id','content_id'])
 worker = Worker(update_metadata,(tmp_df,key))
workers.append(worker)
 worker.start()
for worker in workers:
worker.join()
prev_test_df = test_df
prev_cache = cache<merge>
df = getDaysShift(df)
 df = getChangeGrowth(df)
 df = getMA(df)
 df = getCDSC(df)
 df = getFDSC(df )
%%skip not SUBMIT
env = riiideducation.make_env()
iter_test = env.iter_test()
prev_test_df = None
prev_cache = None
for(test_df, sample_prediction_df)in iter_test:
test_df = test_df.merge(content_df,how='left',on=['content_id',])
 test_df.prior_question_had_explanation = test_df.prior_question_had_explanation.fillna(False ).astype(np.int8)
 test_state = test_df.loc[test_df['content_type_id'] == 0]
test_state.replace([np.inf, -np.inf], 999999999,inplace=True)
 test_state,cache = merge_metadata(test_state)
 test_state['prior_duration'] = test_state.timestamp-test_state.prev_timestamp
test_state = add_mean(test_state)
 test_state['answered_correctly'] = model.predict(test_state[features])
 if prev_test_df is not None:
workers = []
prior_group_answers_correct	= json.loads(test_df.iloc[0].prior_group_answers_correct)
 prior_group_responses = json.loads(test_df.iloc[0].prior_group_responses)
 prior_group_responses = np.array(prior_group_responses)
 prior_group_answers_correct = np.array(prior_group_answers_correct)
 prev_test_df['user_answer'] = prior_group_responses
prev_test_df['answered_correctly'] = prior_group_answers_correct
prev_test_df = prev_test_df[prev_test_df.content_type_id == 0]
user_df = prev_test_df[['user_id','answered_correctly']]
grp = user_df.groupby(['user_id'])
 user_df['user_id_roll_sum'] = grp['answered_correctly'].cumsum()
user_df['user_id_roll_count'] = grp['answered_correctly'].cumcount() +1
user_df = user_df.groupby(['user_id',] ).tail(1)
 user_df.set_index('user_id',inplace=True)
 del user_df['answered_correctly']
win_cols = []
for win in [50,10]:
sum_col = f'user_id_win{win}_sum'
count_col = f'user_id_win{win}_count'
user_df[sum_col] = user_df.user_id_roll_sum
user_df[count_col] = user_df.user_id_roll_count
win_cols.append(sum_col)
 win_cols.append(count_col)
 win_current_state = user_df[win_cols]
user_df = add_previous_state(user_df,('user_id',),prev_cache)
 for win in [50,10]:
sum_col = f'user_id_win{win}_sum'
count_col = f'user_id_win{win}_count'
selector =(user_df[count_col]>win ).values
user_df.loc[selector,[sum_col,count_col]] = win_current_state.loc[selector,[sum_col,count_col]]
tmp_df = prev_test_df[['user_answer','answered_correctly','user_id','timestamp']].groupby('user_id' ).tail(1)
 tmp_df.set_index('user_id',inplace=True)
 tmp_df.columns = [f'prev_{col}' for col in tmp_df.columns]
user_df = user_df.merge(tmp_df,how='left',on=['user_id',])
 worker = Worker(update_metadata,(user_df,['user_id'],))
worker.start()
workers.append(worker)
 cols = ['content_id','bundle_id',]
metadata = user_roll_features(prev_test_df,cols)
 for key in metadata:
col = key[-1]
tmp_df = metadata[key]
win_current_state = tmp_df
tmp_df = add_previous_state(tmp_df,key,prev_cache)
 for win in [50,10]:
sum_col = f'{col}_win{win}_sum'
count_col = f'{col}_win{win}_count'
selector =(tmp_df[count_col]>win ).values
tmp_df.loc[selector,[sum_col,count_col]] = win_current_state.loc[selector,[sum_col,count_col]]
if key ==('user_id','content_id'):
tmp = prev_test_df[['content_id','user_id','answered_correctly',]]
tmp.rename(columns={'answered_correctly':'prev_content_answered_correctly'},inplace=True)
 tmp = tmp.groupby(['content_id','user_id',] ).tail(1)
 tmp.set_index(['user_id','content_id'],inplace=True)
 tmp_df = tmp_df.merge(tmp,how='left',on=['user_id','content_id'])
 worker = Worker(update_metadata,(tmp_df,key))
workers.append(worker)
 worker.start()
for worker in workers:
worker.join()
prev_test_df = test_df
prev_cache = cache
env.predict(test_state[['row_id', 'answered_correctly']])
<install_modules>
df['CSId'] = df.groupby(['Country', 'State'] ).cumcount()
!pip install -q efficientnet >> /dev/null<set_options>
df.loc[df.ForecastId > 0, 'ForecastId'].nunique()
random.seed(a=42)
 os.environ["CUDA_VISIBLE_DEVICES"] = "0"
gpus = tf.config.experimental.list_physical_devices('GPU')
 try:
for gpu in gpus:
tf.config.experimental.set_memory_growth(gpu, True)
 except RuntimeError as e:
print(e)
 CFG = dict(
 batch_size = 32,
read_size = 256,
crop_size = 235,
net_size = 224,
LR = 1e-4,
epochs = 30,
rot = 180.0,
shr = 2.0,
hzoom = 8.0,
wzoom = 8.0,
hshift = 8.0,
wshift = 8.0,
tta_steps = 15,
es_patience = 4, 
)
 GCS_PATH = f'.. /input//melanoma-{CFG["read_size"]}x{CFG["read_size"]}'
files_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')))
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')))
 malig_files = sorted(glob(f'.. /input/malignant-v2-{CFG["read_size"]}x{CFG["read_size"]}/*.tfrec'))
malig_files = malig_files[15:]
files_train = np.concatenate([files_train, malig_files])
<normalization>
df[df.Country =='Italy'][['ConfirmedCases', 'CD_1', 'CC_1']][70:95]
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, cfg):
DIM = cfg["read_size"]
XDIM = DIM%2
rot = cfg['rot'] * tf.random.normal([1], dtype='float32')
 shr = cfg['shr'] * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ cfg['hzoom']
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ cfg['wzoom']
h_shift = cfg['hshift'] * tf.random.normal([1], dtype='float32')
 w_shift = cfg['wshift'] * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3])
 def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'diagnosis' : tf.io.FixedLenFeature([], tf.int64),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['target'], example['image_name']
def read_unlabeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], 0, example['image_name']
def prepare_image(img, cfg=None, augment=True):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.image.resize(img, [cfg['read_size'], cfg['read_size']])
 img = tf.cast(img, tf.float32)/ 255.0
if augment:
img = transform(img, cfg)
 img = tf.image.random_crop(img, [cfg['crop_size'], cfg['crop_size'], 3])
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_hue(img, 0.01)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 else:
img = tf.image.central_crop(img, cfg['crop_size'] / cfg['read_size'])
 img = tf.image.resize(img, [cfg['net_size'], cfg['net_size']])
 img = tf.reshape(img, [cfg['net_size'], cfg['net_size'], 3])
 return img
def get_dataset(files, cfg, augment = False, shuffle = False,
labeled=True):
AUTO = tf.data.experimental.AUTOTUNE
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example),
num_parallel_calls=AUTO)
 ds = ds.map(lambda img, label, fn:(prepare_image(img, augment=augment, cfg=cfg),
label, fn),
num_parallel_calls=AUTO)
 ds = ds.batch(cfg['batch_size'])
 ds = ds.prefetch(AUTO)
 return ds<choose_model_class>
filterwarnings('ignore' )
def get_model(cfg):
model_input = tf.keras.Input(shape=(cfg['net_size'], cfg['net_size'], 3), name='imgIn')
 dummy = tf.keras.layers.Lambda(lambda x:x )(model_input)
 constructor = getattr(efn, f'EfficientNetB0')
 x = constructor(include_top=False, weights='imagenet',
input_shape=(cfg['net_size'], cfg['net_size'], 3),
pooling='avg' )(dummy)
 x = tf.keras.layers.Dense(1, activation='sigmoid', dtype=tf.float32 )(x)
 model = tf.keras.Model(model_input, x)
 return model
def get_opt_loss_fn() :
optimizer = tf.keras.optimizers.Adam(learning_rate=CFG["LR"])
 loss_object = tf.keras.losses.BinaryCrossentropy(from_logits=False)
 return optimizer, loss_object
def get_train_fn() :
@tf.function
def train_step(images, labels, model, optimizer, loss_object):
with tf.GradientTape() as tape:
predictions = model(images, training=True)
 loss = loss_object(labels, predictions[:,0])
 gradients = tape.gradient(loss, model.trainable_variables)
 optimizer.apply_gradients(zip(gradients, model.trainable_variables))
return loss
return train_step
def get_test_fn() :
@tf.function
def test_step(images, labels, model, loss_object):
predictions = model(images, training=False)
 return loss_object(labels, predictions[:,0])
 return test_step
def get_pred_fn() :
@tf.function
def pred_step(images, model):
predictions = model(images, training=False)
 return predictions
return pred_step<split>
cLEncoder = LabelEncoder()
sLEncoder = LabelEncoder()
df.loc[:, 'Country'] = cLEncoder.fit_transform(df.loc[:, 'Country'])
 df.loc[:, 'State'] = sLEncoder.fit_transform(df.loc[:, 'State'] )
fold_cv_scores = []
submission_scores = []
folds = KFold(n_splits=5, shuffle = True, random_state = 42)
 fold_num = 0
for tr_idx, va_idx in folds.split(files_train):
print(f"Starting fold: {fold_num}")
 no_imp = 0
CFG['batch_size'] = 32
checkpoint_filepath = f"checkpoint_{fold_num}.h5"
files_train_tr = files_train[tr_idx]
files_train_va = files_train[va_idx]
ds_train = get_dataset(files_train_tr, CFG, augment=True, shuffle=True)
 ds_val = get_dataset(files_train_va, CFG, augment=False, shuffle=False)
 optimizer, loss_object = get_opt_loss_fn()
model = get_model(CFG)
 train_fn = get_train_fn()
test_fn = get_test_fn()
pred_fn = get_pred_fn()
bestLoss = float("inf")
 for e in range(CFG["epochs"]):
trainLoss = 0
tk0 = tqdm(ds_train)
 for idx,(x,y,_)in enumerate(tk0):
loss = train_fn(x, y, model, optimizer, loss_object)
 trainLoss += loss.numpy()
tk0.set_postfix(loss=trainLoss/(idx+1))
testLoss = 0
tk0 = tqdm(ds_val)
 for idx,(x,y,_)in enumerate(tk0):
loss = test_fn(x, y, model, loss_object)
 testLoss += loss.numpy()
tk0.set_postfix(loss=testLoss/(idx+1))
testLoss /= idx
if testLoss < bestLoss:
no_imp = 0
bestLoss = testLoss
model.save_weights(checkpoint_filepath)
 else:
no_imp += 1
if no_imp > CFG["es_patience"]:
print("Early stopping.. ")
 break
model.load_weights(checkpoint_filepath)
 CFG['batch_size'] = 256
ds_valAug = get_dataset(files_train_va, CFG, augment=True)
 ds_testAug = get_dataset(files_test, CFG, augment=True, labeled=False)
 for t in range(CFG['tta_steps']):
for idx,(x,y,fn)in enumerate(ds_valAug):
predictions = pred_fn(x, model)
 for j in range(predictions.shape[0]):
fold_cv_scores.append([fold_num,
fn[j].numpy().decode("utf-8"),
predictions[j,0].numpy() ,
y[j].numpy() ])
 for idx,(x,y,fn)in enumerate(ds_testAug):
predictions = pred_fn(x, model)
 for j in range(predictions.shape[0]):
submission_scores.append([fold_num,
fn[j].numpy().decode("utf-8"),
predictions[j,0].numpy() ])
 tf.compat.v1.reset_default_graph()
tf.keras.backend.clear_session()
fold_num += 1 
<create_dataframe>
X_Train = df[df.Date <= df_Train.Date.max() ]
X_Train.loc[:, 'Date_Int'] =(X_Train.loc[:, 'Date'].dt.strftime("%m%d")).astype('int16')
 yC_Train = X_Train.CC_1
yF_Train = X_Train.FC_1
X_Train = X_Train.drop(columns=['Id', 'ForecastId', 'Date', 'ConfirmedCases', 'Fatalities', 'CC_1', 'FC_1'])
 print(X_Train.shape, yC_Train.shape, yF_Train.shape )
df_fold = pd.DataFrame(fold_cv_scores, columns=["Fold", "Filename", "Pred", "Label"])
 df_sub = pd.DataFrame(submission_scores, columns=["Fold", "Filename", "Pred"] )<compute_test_metric>
cLGBMR = LGBMRegressor(num_leaves=85,learning_rate=10**-1.89,n_estimators=100,min_sum_hessian_in_leaf=(10**-4.1),min_child_samples=2,subsample=0.97,subsample_freq=10,
colsample_bytree=0.68,reg_lambda=10**1.4,random_state=7,n_jobs=-1)
 fLGBMR = LGBMRegressor(num_leaves=26,learning_rate=10**-1.63,n_estimators=100,min_sum_hessian_in_leaf=(10**-4.04),min_child_samples=14,subsample=0.66,subsample_freq=5,
colsample_bytree=0.8,reg_lambda=10**1.92,random_state=7,n_jobs=-1 )
df_fold = df_fold.groupby(["Filename"] ).mean().reset_index()
print("CV ROCAUC: ")
 print(roc_auc_score(df_fold["Label"], df_fold["Pred"]))<save_to_csv>
cLGBMR.fit(X_Train, yC_Train, categorical_feature=['Country','State'])
 fLGBMR.fit(X_Train, yF_Train, categorical_feature=['Country','State'] )
df_sub = df_sub.groupby(["Filename"] ).mean().reset_index()
df_sub = df_sub[["Filename", "Pred"]]
df_sub.columns=["image_name", "target"]
df_sub.to_csv("submission.csv", index=False )<set_options>
X_Test = df[df.Date >= df_Test.Date.min() ]
X_Test.loc[:, 'Date_Int'] =(X_Test.loc[:, 'Date'].dt.strftime("%m%d")).astype('int16')
 X_Test.drop(columns=['Id', 'ForecastId', 'Date', 'ConfirmedCases', 'Fatalities', 'CC_1', 'FC_1'], axis=1, inplace=True )
%matplotlib inline
print("Tensorflow version " + tf.__version__)
 !pip install -q tfa-nightly
tfa.register_all(custom_kernels=False)
 ! pip install -q efficientnet 
<train_on_grid>
nIds = range(X_Train.CSId.max() +1, X_Test.CSId.max() +1)
 yC_Pred = []
yF_Pred = []
for nId in nIds:
df = getDaysShift(df)
 df = getChangeGrowth(df)
 df = getMA(df)
 df = getCDSC(df)
 df = getFDSC(df)
 df.loc[df.CSId == nId, 'CC_1'] = cLGBMR.predict(X_Test[X_Test.CSId == nId])
 df.loc[df.CSId == nId,'LConfirmedCases'] = df.loc[df.CSId == nId, 'CC_1'] + df.loc[df.CSId == nId, 'CD_1']
df.loc[df.CSId == nId,'ConfirmedCases'] = np.exp(df.loc[df.CSId == nId,'LConfirmedCases'])- 1
df.loc[df.CSId == nId, 'FC_1'] = fLGBMR.predict(X_Test[X_Test.CSId == nId])
 df.loc[df.CSId == nId,'LFatalities'] = df.loc[df.CSId == nId, 'FC_1'] + df.loc[df.CSId == nId, 'FD_1']
df.loc[df.CSId == nId,'Fatalities'] = np.exp(df.loc[df.CSId == nId,'LFatalities'])- 1
DEVICE = 'TPU'
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else: DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print('REPLICAS: {}'.format(REPLICAS))<define_variables>
df.loc[:, 'Country'] = cLEncoder.inverse_transform(df.loc[:, 'Country'])
 df.loc[:, 'State'] = sLEncoder.inverse_transform(df.loc[:, 'State'] )
DATA_PATH = '.. /input/siim-isic-melanoma-classification/'
IMAGE_PATH= DATA_PATH+'jpeg/train/'
INCLUDE_2019_DATA = True
INCLUDE_2018_DATA = True
INCLUDE_CD_MALIGNANT = True
UPSAMPLE = True
DIM = 256
BATCH_SIZE = 32 * REPLICAS
EPOCHS = 20
VERBOSE = 1
TTA = 10
tf.random.set_seed(250688)
 MAX_ROT_ANGLE = 180.0
MAX_SHEAR_LEVEL = 0.1
HSHIFT, VSHIFT = 5., 5.
 MAX_ROT_ANGLE = np.pi*MAX_ROT_ANGLE/180
AUGMENT_FRAC = 0.7
NUM_CUTOUTS = 10
CUTOUT_SIZE = 15
CUTOUT_FRAC = 0.8
LR = 1e-3
LOSS_TYPE = "FOCAL"
GAMMA = 2.
 ALPHA = 0.8
OPT_TYPE = "RAdam"
GCS_PATH_ORG = KaggleDatasets().get_gcs_path("siim-isic-melanoma-classification")
 GCS_PATH = KaggleDatasets().get_gcs_path("melanoma-{}x{}".format(DIM,DIM))
GCS_PATH_2019 = KaggleDatasets().get_gcs_path("isic2019-{}x{}".format(DIM,DIM))
GCS_PATH_M = KaggleDatasets().get_gcs_path("malignant-v2-{}x{}".format(DIM,DIM)) 
<define_variables>
df[df.Country =='Italy'][['ConfirmedCases', 'CC_1', 'Fatalities', 'FC_1', 'CSId']][75:99]
train_files = tf.io.gfile.glob(GCS_PATH+'/train*.tfrec')
 if INCLUDE_2019_DATA:
train_files+= tf.io.gfile.glob([f"{GCS_PATH_2019}/train{(2*x +1):02d}-*.tfrec" for x in range(14)])
 if INCLUDE_2018_DATA:
train_files+= tf.io.gfile.glob([f"{GCS_PATH_2019}/train{(2*x):02d}-*.tfrec" for x in range(15)])
 if INCLUDE_CD_MALIGNANT:
new_malignant_tfrecords = np.arange(15,30 ).tolist()
train_files+= tf.io.gfile.glob([f"{GCS_PATH_M}/train{x}-*.tfrec" for x in new_malignant_tfrecords])
 test_files = tf.io.gfile.glob(GCS_PATH+'/test*.tfrec')
 np.random.shuffle(train_files)
<load_from_csv>
df_out = df.loc[df.ForecastId > 0, ['ForecastId', 'ConfirmedCases', 'Fatalities']]
data = pd.read_csv(DATA_PATH+"train.csv")
 pos = data[data.target==1].shape[0]
neg = data[data.target==0].shape[0]
total = data.shape[0]
weight_for_0 =(1 / neg)*(total)/2.0
weight_for_1 =(1 / pos)*(total)/2.0 
<categorify>
df_out.ForecastId = df_out.ForecastId.astype('int' )
def count_data_items(filenames):

 num_files = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(num_files)
 def decode_image(image_string, label=None):

 image = tf.image.decode_jpeg(image_string, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
if DEVICE== 'TPU':
image = tf.reshape(image, [DIM,DIM, 3])
 if label is not None:
return image, label
else:
return image
def read_labelled_tfrecord(example, return_meta_data = False):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'diagnosis' : tf.io.FixedLenFeature([], tf.int64),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 if return_meta_data:
return example['image'], example['target'],example['sex'], example['age_approx'],example['anatom_site_general_challenge']
else:
return example['image'], example['target']
def read_unlabelled_tfrecord(example, return_image_names = True):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, tfrec_format)
 if return_image_names:
return example['image'], example['image_name']
else:
return example['image']
def crop_resize(x, label=None):
image = tf.image.central_crop(x, central_fraction= 0.7+0.3*tf.random.uniform(shape=[] ).numpy())
 image = tf.image.resize(image, [DIM,DIM])
 return image,label
def do_augmentation(image,label=None):
if(tf.random.uniform([],0,1)<AUGMENT_FRAC):
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_brightness(image, max_delta= 0.05)
 image = tf.image.random_saturation(image, lower=0.5, upper=1.5)
 image = tf.image.random_hue(image, 0.05)
 image = tf.image.random_contrast(image, 0.8, 1.2)
 image = tf.image.resize_with_crop_or_pad(image, tf.random.uniform(shape=[], minval= tf.cast(0.7*DIM, tf.int32), maxval=DIM, dtype=tf.int32), target_width= tf.random.uniform(shape=[], minval=tf.cast(0.7*DIM, tf.int32), maxval=DIM, dtype=tf.int32))
image = tf.image.resize(image, [DIM,DIM])
 shear_x = MAX_SHEAR_LEVEL* tf.random.uniform(shape=[],minval=-1,maxval=1)
 shear_y = MAX_SHEAR_LEVEL * tf.random.uniform(shape=[],minval=-1,maxval=1)
 image = tfa.image.transform(image, [1.0, shear_x, 0.0, shear_y, 1.0, 0.0, 0.0, 0.0])
 image = tfa.image.rotate(image, MAX_ROT_ANGLE * tf.random.uniform([], dtype=tf.float32))
image = tfa.image.translate(image, [HSHIFT * tf.random.uniform(shape=[],minval=-1, maxval=1), VSHIFT * tf.random.uniform(shape=[],minval=-1, maxval=1)])
 if(tf.random.uniform([],0,1)< CUTOUT_FRAC):
image = tfa.image.random_cutout(tf.expand_dims(image, axis=0),mask_size=CUTOUT_SIZE,constant_values= 0)
 for _ in range(NUM_CUTOUTS-1):
image = tfa.image.random_cutout(image,mask_size=CUTOUT_SIZE,constant_values= 0)
 image = tf.clip_by_value(image, 0, 1)
 image = tf.reshape(image, [DIM,DIM, 3])
 if label is not None:
return tf.squeeze(image),label
else:
return tf.squeeze(image)
 def data_loader(filenames, labelled=True, augment=True, repeat=True, shuffle=True,ordered=True,return_image_names=True, batch_size = BATCH_SIZE):

 option_no_order = tf.data.Options()
if not ordered:
option_no_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset.with_options(option_no_order)
 if labelled:
dataset = dataset.map(read_labelled_tfrecord, num_parallel_calls=AUTO)
 else:
dataset = dataset.map(lambda example: read_unlabelled_tfrecord(example,return_image_names=return_image_names), num_parallel_calls=AUTO)
 dataset = dataset.cache()
dataset = dataset.map(decode_image, num_parallel_calls=AUTO)
 if shuffle:
dataset = dataset.shuffle(2048)
 if repeat:
dataset = dataset.repeat()
if augment:
dataset = dataset.map(do_augmentation, num_parallel_calls=AUTO)
 dataset = dataset.batch(batch_size)
 dataset = dataset.prefetch(buffer_size = AUTO)
 return dataset 
<train_model>
df_out.iloc[np.r_[1, 42, 45, 97, 143, 175, 267, 327, 350, 420, 450, 540, 590, 680, 730, 2880, 2900, 2960, 3000, 3050, 7000, 7500, 8000, 8500, 9000, 9500, 10000, 10500, 11000, 11500, 12000, 12500, 13000, -1], :]
for img_batch, label in data_loader(train_files, labelled=True, augment=True ).take(1):
print(img_batch.shape, label)
<categorify>
df_out.to_csv('submission.csv', index=False )
for img, img_name in data_loader(train_files, labelled=False, augment=True, return_image_names=True ).unbatch().take(2):
print(img.shape, img_name.numpy().decode("utf-8"))<load_pretrained>
PATH_WEEK4 = '/kaggle/input/covid19-global-forecasting-week-4'
df_Train = pd.read_csv(f'{PATH_WEEK4}/train.csv', parse_dates=["Date"], engine='python')
 df_Test = pd.read_csv(f'{PATH_WEEK4}/test.csv', parse_dates=["Date"], engine='python')
for img_batch in data_loader(train_files, labelled=False, augment=True, return_image_names=False ).take(1):
print(img_batch.shape )<categorify>
def fillState(state, country):
if state == EMPTY_VAL: return country
return state
c=1
for img,label in data_loader(train_files[:2], labelled=True, shuffle=False, augment=False, repeat=False, ordered=True ).unbatch() :
if c==1:
print(img[2,3].numpy() ,label.numpy())
 c+=1<init_hyperparams>
df_Train.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_Test.rename(columns={'Country_Region':'Country'}, inplace=True)
 EMPTY_VAL = "EMPTY_VAL"
df_Train.rename(columns={'Province_State':'State'}, inplace=True)
 df_Train['State'].fillna(EMPTY_VAL, inplace=True)
 df_Train['State'] = df_Train.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 df_Test.rename(columns={'Province_State':'State'}, inplace=True)
 df_Test['State'].fillna(EMPTY_VAL, inplace=True)
 df_Test['State'] = df_Test.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1 )
def get_lr_callback(PLOT_LR = False):
lr_start = 5e-6
lr_max = 1.4e-6 * BATCH_SIZE
lr_min = 1e-6
lr_ramp_ep = 5
lr_sus_ep = 2
lr_decay = 0.8
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
if PLOT_LR:
plt.figure(figsize=(15, 5))
plt.subplot(1, 2, 1)
 plt.plot([lrfn(e)for e in range(EPOCHS)]);
plt.xlabel('Epoch'); plt.ylabel('LR');
plt.subplot(1, 2, 2);
plt.plot([lrfn(e)for e in range(EPOCHS)]);
plt.xlabel('Epoch'); plt.ylabel('Log LR');
plt.yscale('log');
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback
def build_model(input_shape =(192,192,3), pretrained_model= efn.EfficientNetB2, loss_type=LOSS_TYPE, opt = OPT_TYPE):
inp = tf.keras.layers.Input(shape=input_shape)
 base_model = pretrained_model(include_top=False, weights='imagenet', input_shape=input_shape)
 print("Using {} as the base model".format(base_model.name))
if base_model.name == "resnet50":
for layer in base_model.layers[:143]:
layer.trainable = False
for layer in base_model.layers[143:]:
layer.trainable = True
x = base_model(inp)
 x = tf.keras.layers.GlobalAveragePooling2D()(x)
 x = tf.keras.layers.Dense(256, activation = 'relu' )(x)
 x = tf.keras.layers.Dropout(0.3 )(x)
 x = tf.keras.layers.Dense(128, activation = 'relu' )(x)
 x = tf.keras.layers.Dense(1, activation = 'sigmoid' )(x)
 model = tf.keras.Model(inputs= inp, outputs = x)
 if OPT_TYPE == "Adam":
optimizer = tf.keras.optimizers.Adam(learning_rate= LR)
 elif OPT_TYPE == "RAdam":
optimizer = tfa.optimizers.RectifiedAdam(lr=LR)
 if loss_type == 'BCE':
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05)
 elif loss_type == 'FOCAL':
loss= tfa.losses.SigmoidFocalCrossEntropy(gamma = GAMMA, alpha = ALPHA)
 model.compile(optimizer= optimizer,loss= loss, metrics= ['AUC'])
 return model<set_options>
MIN_TEST_DATE = df_Test.Date.min()
df_train = df_Train.loc[df_Train.Date < MIN_TEST_DATE, :]
y1_Train = df_train.iloc[:, -2]
y2_Train = df_train.iloc[:, -1]
_ = get_lr_callback(PLOT_LR=True )<train_model>
def extractDate(df, colName = 'Date'):

 assert colName in df.columns
df = df.assign(
 week = df.loc[:, colName].dt.week,
month = df.loc[:, colName].dt.month,
day = df.loc[:, colName].dt.day,
dayofweek = df.loc[:, colName].dt.dayofweek,
dayofyear = df.loc[:, colName].dt.dayofyear, 
)
 return df
NUM_TEST_IMAGES = count_data_items(test_files)
 def train_model(kfold_n_splits=5,train_set_aug=True, valid_set_aug=True,test_set_aug=True, pretrained_model= MobileNetV2, display_training_curve = True):
preds = np.zeros(shape=(NUM_TEST_IMAGES,))
oof_labels = []
oof_preds = []
kf = KFold(n_splits=kfold_n_splits,shuffle=True,random_state= 1357)
 if DEVICE=='TPU':
print('using TPU')
 for fold,(idxT,idxV)in enumerate(kf.split(np.arange(15))):
print('
print('
print('
K.clear_session()
with strategy.scope() :
model = build_model(input_shape =(DIM,DIM,3), pretrained_model = pretrained_model)
 train_tfrecords = tf.io.gfile.glob(["{}/train{:02d}-*.tfrec".format(GCS_PATH,i)for i in idxT])
 if INCLUDE_2018_DATA:
train_tfrecords += tf.io.gfile.glob([f"{GCS_PATH_2019}/train{(2*x):02d}-*.tfrec" for x in range(15)])
 if INCLUDE_2019_DATA:
train_tfrecords += tf.io.gfile.glob([f"{GCS_PATH_2019}/train{(2*x +1):02d}-*.tfrec" for x in range(14)])
 if INCLUDE_CD_MALIGNANT:
new_malignant_tfrecords = np.arange(15,30 ).tolist()
train_tfrecords += tf.io.gfile.glob([f"{GCS_PATH_M}/train{x}-*.tfrec" for x in new_malignant_tfrecords])
 if UPSAMPLE:
train_tfrecords += tf.io.gfile.glob([f"{GCS_PATH_M}/train{x:02d}-*.tfrec" for x in idxT])
 np.random.shuffle(train_tfrecords)
 val_tfrecords = tf.io.gfile.glob(["{}/train{:02d}-*.tfrec".format(GCS_PATH,i)for i in idxV])
 train_dataset = data_loader(train_tfrecords, labelled=True, augment=train_set_aug, repeat=True, shuffle=True, ordered=False)
 valid_dataset = data_loader(val_tfrecords, labelled = True, augment=False, repeat=False, shuffle=False, ordered=False)
 NUM_TRAINING_IMAGES = count_data_items(train_tfrecords)
 NUM_VALIDATION_IMAGES = count_data_items(val_tfrecords)
 model_best_epoch = tf.keras.callbacks.ModelCheckpoint(
 "fold{}.h5".format(fold+1), monitor='val_auc', verbose=1, save_best_only=True,
save_weights_only=True, mode='max', save_freq='epoch')
 history = model.fit(train_dataset,epochs = EPOCHS,
steps_per_epoch= np.ceil(NUM_TRAINING_IMAGES/BATCH_SIZE),
verbose= 1, callbacks=[model_best_epoch, get_lr_callback() ],validation_data = valid_dataset)
 print("
model.load_weights("fold{}.h5".format(fold+1))
if valid_set_aug:
print(f"
valid_dataset_tta = data_loader(val_tfrecords, labelled=True, augment=True, shuffle=False, repeat=True)
 valid_images = valid_dataset_tta.map(lambda image, label: image)
 ypred_valid = model.predict(valid_images, steps = np.ceil(TTA*NUM_VALIDATION_IMAGES/BATCH_SIZE), verbose=1)
 ypred_valid = ypred_valid[:NUM_VALIDATION_IMAGES * TTA].reshape(( NUM_VALIDATION_IMAGES,TTA), order = 'F')
 ypred_valid = ypred_valid.mean(axis = 1)
 else:
print('
valid_dataset_no_tta = data_loader(val_tfrecords, labelled=True, augment=False, shuffle=False,repeat=False)
 valid_images = valid_dataset_no_tta.map(lambda image, label: image)
 ypred_valid = model.predict(valid_images,steps = NUM_VALIDATION_IMAGES/BATCH_SIZE, verbose =1)
 valid_labels = np.array([label.numpy() for label in data_loader(val_tfrecords, labelled=True, augment=False, shuffle=False,repeat=False ).map(lambda image, label: label ).unbatch() ])
 auc_valid = roc_auc_score(valid_labels, ypred_valid)
 print(f'AUC of validation fold {fold+1}(with TTA = {TTA})= {auc_valid}')
 oof_labels.append(valid_labels)
 oof_preds.append(ypred_valid)
 if test_set_aug:
print(f"
test_data = data_loader(test_files, labelled=False, augment=True, shuffle=False, repeat=True, return_image_names=False, batch_size = BATCH_SIZE*8)
 ypred_test = model.predict(test_data, steps = TTA*NUM_TEST_IMAGES/(BATCH_SIZE*8), verbose=1)
 ypred_test = ypred_test[:NUM_TEST_IMAGES * TTA].reshape(( NUM_TEST_IMAGES,TTA), order = 'F')
 ypred_test = ypred_test.mean(axis = 1)
 else:
print("
test_data = data_loader(test_files, labelled=False, augment=False, shuffle=False, repeat=False, return_image_names=False)
 ypred_test = model.predict(test_data, verbose=1)
 preds +=(ypred_test/kfold_n_splits)
 if display_training_curve:
print("Fold: {}".format(fold+1))
display_training_curves(history.history['loss'], history.history['val_loss'], 'loss', 211)
 display_training_curves(history.history['auc'], history.history['val_auc'], 'AUC', 212)
 return preds, oof_labels,oof_preds<train_model>
def createNewDataset(df):

 df_New = df.copy()
df_New.loc[:, 'Date_Int'] =(df_New.loc[:, 'Date'].dt.strftime("%m%d")).astype('int16')
 df_New.drop(columns=['Date'], axis=1, inplace=True)
 return df_New
pretrained_model = efn.EfficientNetB2
preds, oof_labels,oof_preds = train_model(train_set_aug=True,pretrained_model = pretrained_model, display_training_curve=True )<compute_test_metric>
X_Train = createNewDataset(df_train)
 X_Test = createNewDataset(df_Test )
roc_auc_score(np.concatenate(oof_labels),np.concatenate(oof_preds))<save_to_csv>
days_change = [1, 2, 3, 5, 7, 10]
def getChangeGrowth(df):
newCCols = []
newGCols = []
for day in days_change:
newCCol = f'C_{day}'
df['C'+newCCol] = df['LConfirmedCases'] - df[f'CD_{day}']
df['F'+newCCol] = df['LFatalities'] - df[f'FD_{day}']
newCCols.append(newCCol)
 newGCol = f'G_{day}'
df['C'+newGCol] = df['C'+newCCol] / df[f'CD_{day}']
df['F'+newGCol] = df['F'+newCCol] / df[f'FD_{day}']
newGCols.append(newGCol)
 df.fillna(0, inplace=True)
 return df
test_img_names = []
test_dataset = data_loader(test_files, labelled=False, augment=False, shuffle=False, repeat=False, return_image_names=True, ordered=True)
 for _,image_name in test_dataset.unbatch() :
test_img_names.append(image_name.numpy().decode("utf-8"))
sample_sub = pd.read_csv(DATA_PATH+'/sample_submission.csv')
 sample_sub.loc[:,'image_name' ] = test_img_names
sample_sub.loc[:,'target' ] = preds
sample_sub.to_csv("submission.csv",index=False )<save_to_csv>
windows = [1, 2, 3, 5, 7]
def getMA(df):
newCMACols = []
newGMACols = []
for window in windows:
for day in days_change:
newCMACol = f'CMA_{day}_{window}'
df['C'+newCMACol] = df[f'CC_{day}'].rolling(window ).mean()
df['F'+newCMACol] = df[f'FC_{day}'].rolling(window ).mean()
newCMACols.append(newCMACol)
 newGMACol = f'GMA_{day}_{window}'
df['C'+newGMACol] = df[f'CG_{day}'].rolling(window ).mean()
df['F'+newGMACol] = df[f'FG_{day}'].rolling(window ).mean()
newGMACols.append(newGMACol)
 df.fillna(0, inplace=True)
 return df
sub = pd.read_csv('.. /input/best-private-sub/effnet6_image_chrisgroupkfold_fastplatoschedule_cutoutshiftsrotates_customaugs_denseelu_mdropout_difflrs_distributed_bigbatch_nodupls_384res_extrenaldata1817_batchacum4_SWA_6ttamicroscope6randomttas_meanfolds.csv')
 sub.to_csv('submission.csv', index=False )<import_modules>
cases = [1, 50, 100, 500, 1000, 5000, 35000, 75000, 100000]
def getCDSC(df):
newCDSCCols = []
for case in cases:
newDSCCol = f'{case}_CDSC'
df.loc[df.CD_1 == 0, newDSCCol] = 0
df.loc[df.CD_1 >= case, newDSCCol] = df[df.CD_1 >= case].groupby(['Country', 'State'] ).cumcount()
newCDSCCols.append(newDSCCol)
 df.fillna(0, inplace=True)
 return df
import pandas as pd, numpy as np, os
import matplotlib.pyplot as plt
from scipy.optimize import minimize, fmin
from sklearn.metrics import roc_auc_score
from bayes_opt import BayesianOptimization
from functools import partial
from sklearn import metrics<compute_test_metric>
deaths = [1, 50, 100, 500, 1000, 5000, 35000]
def getFDSC(df):
newFDSCCols = []
for death in deaths:
newDSCCol = f'{death}_FDSC'
df.loc[df.FD_1 == 0, newDSCCol] = 0
df.loc[df.FD_1 >= death, newDSCCol] = df[df.FD_1 >= death].groupby(['Country', 'State'] ).cumcount()
newFDSCCols.append(newDSCCol)
 df.fillna(0, inplace=True)
 return df
class OptimizeAUC:

 def __init__(self):
self.coef_ = 0
def _auc(self, coef, X, y):

 x_coef = X * coef
predictions = np.sum(x_coef, axis=1)
 auc_score = metrics.roc_auc_score(y, predictions)
 return -1.0 * auc_score
def fit(self, X, y):
loss_partial = partial(self._auc, X=X, y=y)
 initial_coef = np.random.dirichlet(np.ones(X.shape[1]), size=1)
 self.coef_ = fmin(loss_partial, initial_coef, disp=True)
 def predict(self, X):
x_coef = X * self.coef_
predictions = np.sum(x_coef, axis=1)
 return predictions<load_from_csv>
df = pd.concat([df_Train, df_Test[df_Test.Date > df_Train.Date.max() ]], axis=0, sort=False, ignore_index=True )
sample_submission = pd.read_csv(".. /input/siim-isic-melanoma-classification/sample_submission.csv" )<define_variables>
df['LConfirmedCases'] = np.log1p(df['ConfirmedCases'])
 df['LFatalities'] = np.log1p(df['Fatalities'] )
SUB_DIR = ".. /input/siimsubmitionfiles/Final_Submissions/Final_Submissions"
TF_0OFS = ".. /input/siimsubmitionfiles/TF-OOFS/TF-OOFS"
TF_SUBS = ".. /input/siimsubmitionfiles/TF-SUBS/TF-SUBS"<load_from_csv>
df.loc[(df.Date >= df_Test.Date.min())&(df.Date <= df_Train.Date.max()), 'ForecastId'] = df_Test.loc[(df_Test.Date >= df_Test.Date.min())&(df_Test.Date <= df_Train.Date.max()), 'ForecastId'].values
offs_b5_df = pd.read_csv(f"{SUB_DIR}/offs_b5_df.csv")
 offs_b5_ns_df = pd.read_csv(f"{SUB_DIR}/offs_b5_ns_df.csv")
 offs_b5_ds_df = pd.read_csv(f"{SUB_DIR}/offs_b5_512_ds_df.csv")
 offs_b5_10_ds_df = pd.read_csv(f"{SUB_DIR}/offs_b5_10_ds_df.csv")
 offs_b5_f_ds_df = pd.read_csv(f"{SUB_DIR}/offs_b5_f_ds_df.csv")
 offs_tf_b5_df = pd.read_csv(f"{SUB_DIR}/offs_tf_b5_df.csv")
 offs_tf_b5_v1_df = pd.read_csv(f"{SUB_DIR}/offs_tf_b5_v1_df.csv")
 offs_b6_df = pd.read_csv(f"{SUB_DIR}/offs_b6_df.csv")
 offs_b6_2_df = pd.read_csv(f"{SUB_DIR}/offs_b6_2_ds_df.csv")
 offs_b6_10_df = pd.read_csv(f"{SUB_DIR}/offs_b6_10_ds_df.csv")
 offs_tf_b6_df = pd.read_csv(f"{SUB_DIR}/offs_tf_b6_df.csv")
 offs_d201_df = pd.read_csv(f"{SUB_DIR}/offs_d201_df.csv")
 offs_d201_ns_df = pd.read_csv(f"{SUB_DIR}/offs_d201_ns_df.csv")
 offs_d201_512_df = pd.read_csv(f"{SUB_DIR}/offs_d201_512_df.csv")
 offs_d201_512_ds_df = pd.read_csv(f"{SUB_DIR}/offs_d201_512_ds_df.csv")
 offs_tf_d201_df = pd.read_csv(f"{SUB_DIR}/offs_tf_d201_df.csv")
 offs_b4_df = pd.read_csv(f"{SUB_DIR}/offs_b4_df.csv")
 offs_b4_f_df = pd.read_csv(f"{SUB_DIR}/offs_b4_f_5_ds_df.csv")
 offs_b7_224_df = pd.read_csv(f"{SUB_DIR}/offs_b7_224_ds_df.csv")
 offs_tf_b7_df = pd.read_csv(f"{SUB_DIR}/offs_tf_b7_df.csv")
 offs_b7_512_ds_df = pd.read_csv(f"{SUB_DIR}/offs_b7_512_ds_df.csv")
 offs_dpn92_ds_df = pd.read_csv(f"{SUB_DIR}/offs_dpn92_ds_df.csv")
 offs_sk50_ds_df = pd.read_csv(f"{SUB_DIR}/offs_sk50_5_ds_df.csv")
 offs_meta_df = pd.read_csv(f"{SUB_DIR}/tabular_oof_df.csv" ).sort_values("image_name")
 offs_meta_df = offs_meta_df[offs_meta_df.image_name.isin(offs_dpn92_ds_df.image_name)].reset_index(drop=True)
 offs_meta_2_df = pd.read_csv(f"{SUB_DIR}/tabular2_oof_df.csv" ).sort_values("image_name")
 offs_meta_2_df = offs_meta_2_df[offs_meta_2_df.image_name.isin(offs_dpn92_ds_df.image_name)].reset_index(drop=True )<create_dataframe>
df = getDaysShift(df)
 df = getChangeGrowth(df)
 df = getMA(df)
 df = getCDSC(df)
 df = getFDSC(df )
offs_final_df = pd.DataFrame({
"image_name" : offs_b5_df.image_name.values,
"target" : offs_b5_df.target.values,
"b5" : offs_b5_df.prediction.values,
"b5_ns": offs_b5_ns_df.prediction.values,
"b5_ds": offs_b5_ds_df.prediction.values,
"b5_ds_10": offs_b5_10_ds_df.prediction.values,
"b5_f_ds" : offs_b5_f_ds_df.prediction.values,
"tf_b5" : offs_tf_b5_df.pred.values,
"tf_b5_v1": offs_tf_b5_v1_df.pred.values,
"b6" : offs_b6_df.prediction.values,
"b6_2" : offs_b6_2_df.prediction.values,
"b6_10" : offs_b6_10_df.prediction,
"tf_b6": offs_tf_b6_df.pred.values,
"b4" : offs_b4_df.prediction.values,
"b4_f" : offs_b4_f_df.prediction.values,
"d201" :offs_d201_df.prediction.values,
"d201_ns" : offs_d201_ns_df.prediction.values,
"d201_512" : offs_d201_512_df.prediction.values,
"d201_512_ds" : offs_d201_512_ds_df.prediction.values,
"tf_d201" : offs_tf_d201_df.pred.values,
"b7_224_ds" : offs_b7_224_df.prediction.values,
"tf_b7" : offs_tf_b7_df.pred.values,
"b7_512_ds" : offs_b7_512_ds_df.prediction.values,
"dpn92_ds" : offs_dpn92_ds_df.prediction.values,
"sk50_ds" : offs_sk50_ds_df.prediction.values,
"meta_sub" : offs_meta_df.prediction.values,
"meta_sub_2" : offs_meta_2_df.prediction.values
} )<prepare_x_and_y>
df['CSId'] = df.groupby(['Country', 'State'] ).cumcount()
train_df = offs_final_df
train_cols = [
'b5', 'b5_ns', 'b5_ds', 'b5_ds_10', 'b5_f_ds','tf_b5', 'tf_b5_v1',
'b6', 'b6_2', 'b6_10', 'tf_b6',
'b4', 'b4_f',
'd201', 'd201_ns', 'd201_512', 'd201_512_ds', 'tf_d201',
'b7_224_ds','tf_b7', 'b7_512_ds',
'dpn92_ds',
'sk50_ds',
'meta_sub', 'meta_sub_2'
]
X = train_df[train_cols]
y = train_df["target"]<train_model>
df.loc[df.ForecastId > 0, 'ForecastId'].nunique()
opt = OptimizeAUC()
opt.fit(X, y )<load_from_csv>
df[df.Country =='Italy'][['ConfirmedCases', 'CD_1', 'CC_1']][70:95]
b5_512_5 = pd.read_csv(f"{SUB_DIR}/Final_B5_512_456_5.csv")["target"]
b5_512_5_ns = pd.read_csv(f"{SUB_DIR}/Final_NS_E5_512_456_5.csv")["target"]
b5_512_5_ds = pd.read_csv(f"{SUB_DIR}/Final_DS_B5_512_456_5.csv")["target"]
b5_512_10_ds = pd.read_csv(f"{SUB_DIR}/Final_DS_B5_512_456_10.csv")["target"]
b5_512_f_ds = pd.read_csv(f"{SUB_DIR}/F_Final_DS_B5_512_456_5.csv")["target"]
b5_512_5_tf = pd.read_csv(f"{SUB_DIR}/TF-B5-512-V.csv")["target"]
b5_512_5_v1 = pd.read_csv(f"{SUB_DIR}/TF-B5-512.csv")["target"]
b6_528_5 = pd.read_csv(f"{SUB_DIR}/Final_B6_768_528_5.csv")["target"]
b6_512_2 = pd.read_csv(f"{SUB_DIR}/Final_DSO_B6_768_528_2.csv")["target"]
b6_512_10 = pd.read_csv(f"{SUB_DIR}/Final_DS_B6_768_528_10.csv")["target"]
b6_512_5_tf = pd.read_csv(f"{SUB_DIR}/TF-B6-512-5.csv")["target"]
b4_300_10 = pd.read_csv(f"{SUB_DIR}/Final_B4_512_380_10.csv")["target"]
b4_f = pd.read_csv(f"{SUB_DIR}/F_Final_DS_B4_512_380_5.csv")["target"]
d201_224_5 = pd.read_csv(f"{SUB_DIR}/Final_D201_512_224_5.csv")["target"]
d201_224_5_ns = pd.read_csv(f"{SUB_DIR}/Final_NS_D201_512_224_5.csv")["target"]
d201_512_5 = pd.read_csv(f"{SUB_DIR}/Final_D201_512_512_5.csv")["target"]
d201_512_5_ds = pd.read_csv(f"{SUB_DIR}/Final_DS_D201_512_512_5.csv")["target"]
d201_512_5_tf = pd.read_csv(f"{SUB_DIR}/TF-D201-512-5.csv")["target"]
b7_224_ds = pd.read_csv(f"{SUB_DIR}/Final_DS_B7_512_224_5.csv")["target"]
b7_512_ds = pd.read_csv(f"{SUB_DIR}/Final_DS_B7_512_512_5.csv")["target"]
b7_512_tf = pd.read_csv(f"{SUB_DIR}/TF-B7-512-5.csv")["target"]
dpn92_ds = pd.read_csv(f"{SUB_DIR}/Final_DS_dpn92_512_512_5.csv")["target"]
sk50_ds = pd.read_csv(f"{SUB_DIR}/Final_DS_sk50_512_512_5.csv")["target"]
meta_1 = pd.read_csv(f"{SUB_DIR}/tabular_test_prediction.csv")["target"]
meta_2 = pd.read_csv(f"{SUB_DIR}/tabular2_test_prediction.csv")["target"]<load_from_csv>
filterwarnings('ignore' )
W = opt.coef_
pytorch_sub = np.mean([
W[0] * b5_512_5,
W[1] * b5_512_5_ns,
W[2] * b5_512_5_ds,
W[3] * b5_512_10_ds,
W[4] * b5_512_f_ds,
W[5] * b5_512_5_tf,
W[6] * b5_512_5_v1,
W[7] * b6_528_5,
W[8] * b6_512_2,
W[9] * b6_512_10,
W[10] * b6_512_5_tf,
W[11] * b4_300_10,
W[12] * b4_f,
W[13] * d201_224_5,
W[14] * d201_224_5_ns,
W[15] * d201_512_5,
W[16] * d201_512_5_ds,
W[17] * d201_512_5_tf,
W[18] * b7_224_ds,
W[19] * b7_512_ds,
W[20] * b7_512_tf,
W[21] * dpn92_ds,
W[22] * sk50_ds,
W[23] * meta_1,
W[24] * meta_2,
], axis=0 )<save_to_csv>
cLEncoder = LabelEncoder()
sLEncoder = LabelEncoder()
df.loc[:, 'Country'] = cLEncoder.fit_transform(df.loc[:, 'Country'])
 df.loc[:, 'State'] = sLEncoder.fit_transform(df.loc[:, 'State'] )
sample_submission["target"] = pytorch_sub
sample_submission.to_csv("Final_Pytorch.csv", index=False )<load_from_csv>
X_Train = df[df.Date <= df_Train.Date.max() ]
X_Train.loc[:, 'Date_Int'] =(X_Train.loc[:, 'Date'].dt.strftime("%m%d")).astype('int16')
 yC_Train = X_Train.CC_1
yF_Train = X_Train.FC_1
X_Train = X_Train.drop(columns=['Id', 'ForecastId', 'Date', 'ConfirmedCases', 'Fatalities', 'CC_1', 'FC_1'])
 print(X_Train.shape, yC_Train.shape, yF_Train.shape )
offs_b7_512_15 = pd.read_csv(f"{TF_0OFS}/offs_b7_512_15.csv")
 offs_b6_512_15 = pd.read_csv(f"{TF_0OFS}/offs_b6_512_15.csv")
 offs_b5_512_15 = pd.read_csv(f"{TF_0OFS}/offs_b5_512_15.csv")
 offs_b7_512_10 = pd.read_csv(f"{TF_0OFS}/offs_b7_512_10.csv")
 offs_b6_512_10 = pd.read_csv(f"{TF_0OFS}/offs_b6_512_10.csv")
 offs_b5_512_10 = pd.read_csv(f"{TF_0OFS}/offs_b5_512_10.csv")
 offs_b4_512_10 = pd.read_csv(f"{TF_0OFS}/offs_b4_512_10.csv")
 offs_b3_512_10 = pd.read_csv(f"{TF_0OFS}/offs_b3_512_10.csv")
 offs_b2_512_10 = pd.read_csv(f"{TF_0OFS}/offs_b2_512_10.csv")
 offs_b1_512_10 = pd.read_csv(f"{TF_0OFS}/offs_b1_512_10.csv")
 offs_b7_768_5 = pd.read_csv(f"{TF_0OFS}/offs_b7_768_5.csv")
 offs_b6_768_5 = pd.read_csv(f"{TF_0OFS}/offs_b6_768_5.csv")
 offs_b5_768_5 = pd.read_csv(f"{TF_0OFS}/offs_b5_768_5.csv")
 offs_b4_768_5 = pd.read_csv(f"{TF_0OFS}/offs_b4_768_5.csv")
 offs_b3_768_5 = pd.read_csv(f"{TF_0OFS}/offs_b3_768_5.csv")
 offs_b2_768_5 = pd.read_csv(f"{TF_0OFS}/offs_b2_768_5.csv")
 offs_b1_768_5 = pd.read_csv(f"{TF_0OFS}/offs_b1_768_5.csv")
 offs_b7_384_15 = pd.read_csv(f"{TF_0OFS}/offs_b7_384_15.csv")
 offs_b6_384_15 = pd.read_csv(f"{TF_0OFS}/offs_b6_384_15.csv")
 offs_b5_384_15 = pd.read_csv(f"{TF_0OFS}/offs_b5_384_15.csv" )<create_dataframe>
cLGBMR = LGBMRegressor(num_leaves=85,learning_rate=10**-1.89,n_estimators=100,min_sum_hessian_in_leaf=(10**-4.1),min_child_samples=2,subsample=0.97,subsample_freq=10,
colsample_bytree=0.68,reg_lambda=10**1.4,random_state=7,n_jobs=-1)
 fLGBMR = LGBMRegressor(num_leaves=26,learning_rate=10**-1.63,n_estimators=100,min_sum_hessian_in_leaf=(10**-4.04),min_child_samples=14,subsample=0.66,subsample_freq=5,
colsample_bytree=0.8,reg_lambda=10**1.92,random_state=7,n_jobs=-1 )
offs_final_tf_df = pd.DataFrame({
"target" : offs_b7_512_15.target.values,
"image_name" : offs_b7_512_15.image_name.values,
"b7_512_15" : offs_b7_512_15.prediction.values,
"b6_512_15" : offs_b6_512_15.prediction.values,
"b5_512_15" : offs_b5_512_15.prediction.values,
"b7_512_10" : offs_b7_512_10.prediction.values,
"b6_512_10" : offs_b6_512_10.prediction.values,
"b5_512_10" : offs_b5_512_10.prediction.values,
"b4_512_10" : offs_b4_512_10.prediction.values,
"b3_512_10" : offs_b3_512_10.prediction.values,
"b2_512_10" : offs_b2_512_10.prediction.values,
"b1_512_10" : offs_b1_512_10.prediction.values,
"b7_768_5" : offs_b7_768_5.prediction.values,
"b6_768_5" : offs_b6_768_5.prediction.values,
"b5_768_5" : offs_b5_768_5.prediction.values,
"b4_768_5" : offs_b4_768_5.prediction.values,
"b3_768_5" : offs_b3_768_5.prediction.values,
"b2_768_5" : offs_b2_768_5.prediction.values,
"b1_768_5" : offs_b1_768_5.prediction.values,
"b7_384_15" : offs_b7_384_15.prediction.values,
"b6_384_15" : offs_b6_384_15.prediction.values,
"b5_384_15" : offs_b5_384_15.prediction.values,
} )<prepare_x_and_y>
cLGBMR.fit(X_Train, yC_Train, categorical_feature=['Country','State'])
 fLGBMR.fit(X_Train, yF_Train, categorical_feature=['Country','State'] )
train_df = offs_final_tf_df
train_cols = [
'b7_512_15', 'b6_512_15', 'b5_512_15',
'b7_512_10', 'b6_512_10', 'b5_512_10', 'b4_512_10', 'b3_512_10',
'b2_512_10', 'b1_512_10', 'b7_768_5', 'b6_768_5', 'b5_768_5',
'b4_768_5', 'b3_768_5', 'b2_768_5', 'b1_768_5', 'b7_384_15',
'b6_384_15', 'b5_384_15'
]
X = train_df[train_cols]
y = train_df["target"]<load_from_csv>
X_Test = df[df.Date >= df_Test.Date.min() ]
X_Test.loc[:, 'Date_Int'] =(X_Test.loc[:, 'Date'].dt.strftime("%m%d")).astype('int16')
 X_Test.drop(columns=['Id', 'ForecastId', 'Date', 'ConfirmedCases', 'Fatalities', 'CC_1', 'FC_1'], axis=1, inplace=True )
B7_512_15 = pd.read_csv(f"{TF_SUBS}/B7_512_15.csv")["target"]
B6_512_15 = pd.read_csv(f"{TF_SUBS}/B6_512_15.csv")["target"]
B5_512_15 = pd.read_csv(f"{TF_SUBS}/B5_512_15.csv")["target"]
B7_512_10 = pd.read_csv(f"{TF_SUBS}/B7_512_10.csv")["target"]
B6_512_10 = pd.read_csv(f"{TF_SUBS}/B6_512_10.csv")["target"]
B5_512_10 = pd.read_csv(f"{TF_SUBS}/B5_512_10.csv")["target"]
B4_512_10 = pd.read_csv(f"{TF_SUBS}/B4_512_10.csv")["target"]
B3_512_10 = pd.read_csv(f"{TF_SUBS}/B3_512_10.csv")["target"]
B2_512_10 = pd.read_csv(f"{TF_SUBS}/B2_512_10.csv")["target"]
B1_512_10 = pd.read_csv(f"{TF_SUBS}/B1_512_10.csv")["target"]
B7_768_5 = pd.read_csv(f"{TF_SUBS}/B7_768_5.csv")["target"]
B6_768_5 = pd.read_csv(f"{TF_SUBS}/B6_768_5.csv")["target"]
B5_768_5 = pd.read_csv(f"{TF_SUBS}/B5_768_5.csv")["target"]
B4_768_5 = pd.read_csv(f"{TF_SUBS}/B4_768_5.csv")["target"]
B3_768_5 = pd.read_csv(f"{TF_SUBS}/B3_768_5.csv")["target"]
B2_768_5 = pd.read_csv(f"{TF_SUBS}/B2_768_5.csv")["target"]
B1_768_5 = pd.read_csv(f"{TF_SUBS}/B1_768_5.csv")["target"]
B7_384_15 = pd.read_csv(f"{TF_SUBS}/B7_384_15.csv")["target"]
B6_384_15 = pd.read_csv(f"{TF_SUBS}/B6_384_15.csv")["target"]
B5_384_15 = pd.read_csv(f"{TF_SUBS}/B5_384_15.csv")["target"]<load_from_csv>
nIds = range(X_Train.CSId.max() +1, X_Test.CSId.max() +1)
 yC_Pred = []
yF_Pred = []
for nId in nIds:
df = getDaysShift(df)
 df = getChangeGrowth(df)
 df = getMA(df)
 df = getCDSC(df)
 df = getFDSC(df)
 df.loc[df.CSId == nId, 'CC_1'] = cLGBMR.predict(X_Test[X_Test.CSId == nId])
 df.loc[df.CSId == nId,'LConfirmedCases'] = df.loc[df.CSId == nId, 'CC_1'] + df.loc[df.CSId == nId, 'CD_1']
df.loc[df.CSId == nId,'ConfirmedCases'] = np.exp(df.loc[df.CSId == nId,'LConfirmedCases'])- 1
df.loc[df.CSId == nId, 'FC_1'] = fLGBMR.predict(X_Test[X_Test.CSId == nId])
 df.loc[df.CSId == nId,'LFatalities'] = df.loc[df.CSId == nId, 'FC_1'] + df.loc[df.CSId == nId, 'FD_1']
df.loc[df.CSId == nId,'Fatalities'] = np.exp(df.loc[df.CSId == nId,'LFatalities'])- 1
W = opt.coef_
tf_sub = np.mean([
W[0] * B7_512_15,
W[1] * B6_512_15,
W[2] * B5_512_15,
W[3] * B7_512_10,
W[4] * B6_512_10,
W[5] * B5_512_10,
W[6] * B4_512_10,
W[7] * B3_512_10,
W[8] * B2_512_10,
W[9] * B1_512_10,
W[10] * B7_768_5,
W[11] * B6_768_5,
W[12] * B5_768_5,
W[13] * B4_768_5,
W[14] * B3_768_5,
W[15] * B2_768_5,
W[16] * B1_768_5,
W[17] * B7_384_15,
W[18] * B6_384_15,
W[19] * B5_384_15,
], axis=0 )<save_to_csv>
df.loc[:, 'Country'] = cLEncoder.inverse_transform(df.loc[:, 'Country'])
 df.loc[:, 'State'] = sLEncoder.inverse_transform(df.loc[:, 'State'] )
sample_submission["target"] = tf_sub
sample_submission.to_csv("Final_TF.csv", index=False )<merge>
df[df.Country =='Italy'][['ConfirmedCases', 'CC_1', 'Fatalities', 'FC_1', 'CSId']][75:99]
train_df = pd.merge(offs_final_df, offs_final_tf_df, how="left", left_on=["image_name", "target"], right_on=["image_name", "target"])
 train_cols = [
'b5', 'b5_ns', 'b5_ds', 'b5_ds_10', 'b5_f_ds',
'tf_b5', 'tf_b5_v1', 'b6', 'b6_2', 'b6_10', 'tf_b6', 'b4', 'b4_f',
'd201', 'd201_ns', 'd201_512', 'd201_512_ds', 'tf_d201', 'b7_224_ds',
'tf_b7', 'b7_512_ds', 'dpn92_ds', 'sk50_ds',
'b7_512_15', 'b6_512_15', 'b5_512_15', 'b7_512_10', 'b6_512_10',
'b5_512_10', 'b4_512_10', 'b3_512_10', 'b2_512_10', 'b1_512_10',
'b7_768_5', 'b6_768_5', 'b5_768_5', 'b4_768_5', 'b3_768_5', 'b2_768_5',
'b1_768_5', 'b7_384_15', 'b6_384_15', 'b5_384_15'
]
X = train_df[train_cols]
y = train_df["target"]<train_model>
df_out = df.loc[df.ForecastId > 0, ['ForecastId', 'ConfirmedCases', 'Fatalities']]
opt = OptimizeAUC()
opt.fit(X, y )<load_from_csv>
df_out.ForecastId = df_out.ForecastId.astype('int' )
W = opt.coef_
pytorch_tf_sub = np.mean([
W[0] * b5_512_5,
W[1] * b5_512_5_ns,
W[2] * b5_512_5_ds,
W[3] * b5_512_10_ds,
W[4] * b5_512_f_ds,
W[5] * b5_512_5_tf,
W[6] * b5_512_5_v1,
W[7] * b6_528_5,
W[8] * b6_512_2,
W[9] * b6_512_10,
W[10] * b6_512_5_tf,
W[11] * b4_300_10,
W[12] * b4_f,
W[13] * d201_224_5,
W[14] * d201_224_5_ns,
W[15] * d201_512_5,
W[16] * d201_512_5_ds,
W[17] * d201_512_5_tf,
W[18] * b7_224_ds,
W[19] * b7_512_ds,
W[20] * b7_512_tf,
W[21] * dpn92_ds,
W[22] * sk50_ds,
W[23] * B7_512_15,
W[24] * B6_512_15,
W[25] * B5_512_15,
W[26] * B7_512_10,
W[27] * B6_512_10,
W[38] * B5_512_10,
W[39] * B4_512_10,
W[30] * B3_512_10,
W[31] * B2_512_10,
W[32] * B1_512_10,
W[33] * B7_768_5,
W[34] * B6_768_5,
W[35] * B5_768_5,
W[36] * B4_768_5,
W[37] * B3_768_5,
W[38] * B2_768_5,
W[39] * B1_768_5,
W[40] * B7_384_15,
W[41] * B6_384_15,
W[42] * B5_384_15,
], axis=0 )<save_to_csv>
df_out.iloc[np.r_[1, 42, 45, 97, 143, 175, 267, 327, 350, 420, 450, 540, 590, 680, 730, 2880, 2900, 2960, 3000, 3050, 7000, 7500, 8000, 8500, 9000, 9500, 10000, 10500, 11000, 11500, 12000, 12500, 13000, -1], :]
<load_from_csv><EOS> 
df_out.to_csv('submission.csv', index=False )