Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<train_on_grid>
!pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index
!pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index
reg_case_f=ElasticNet(random_state=42,l1_ratio=0.1,max_iter=4500)
 params = [{'alpha': [10**-4,10**-3, 10**-2,10**-1, 10**0,10**1, 10**2,10**3,10**4]}]
clf_f= RandomizedSearchCV(reg_case_f, params, cv=4, scoring='neg_root_mean_squared_error',return_train_score=True)
 search_f=clf_f.fit(X_train_cases_f, y_train_cases_f)
 results_f = pd.DataFrame.from_dict(clf_f.cv_results_ )<train_model>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )
best_alpha=10000
best_itr=4500
final_reg_case_f=ElasticNet(random_state=42,alpha=best_alpha,l1_ratio=0.1,max_iter=best_itr)
 final_reg_case_f.fit(X_train_cases_f,y_train_cases_f )<predict_on_test>
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=config )
pred_f=final_reg_case_f.predict(X_test_cases_f)
 print("RMSE is:",(mean_squared_error(y_test_cases_f,pred_f)) **0.5 )<train_on_grid>
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv' )
reg_fatal_f=ElasticNet(random_state=42,alpha=best_alpha,l1_ratio=0.1,max_iter=2500)
 params = [{'alpha': [10**-4,10**-3, 10**-2,10**-1, 10**0,10**1, 10**2,10**3,10**4]}]
clf_f= RandomizedSearchCV(reg_fatal_f, params, cv=4, scoring='neg_root_mean_squared_error',return_train_score=True)
 search_f=clf_f.fit(X_train_fatal_f, y_train_fatal_f)
 results_f = pd.DataFrame.from_dict(clf_f.cv_results_ )<train_model>
def get_tab(df):
vector = [(df.Age.values[0] - 30)/ 30]
if df.Sex.values[0] == 'male':
vector.append(0)
 else:
vector.append(1)
 if df.SmokingStatus.values[0] == 'Never smoked':
vector.extend([0,0])
 elif df.SmokingStatus.values[0] == 'Ex-smoker':
vector.extend([1,1])
 elif df.SmokingStatus.values[0] == 'Currently smokes':
vector.extend([0,1])
 else:
vector.extend([1,0])
 return np.array(vector )
best_alpha=100
best_itr=2700
final_reg_fatal_f=ElasticNet(random_state=42,alpha=best_alpha,l1_ratio=0.1,max_iter=best_itr)
 final_reg_fatal_f.fit(X_train_fatal_f,y_train_fatal_f )<predict_on_test>
A = {}
TAB = {}
P = []
for i, p in tqdm(enumerate(train.Patient.unique())) :
sub = train.loc[train.Patient == p, :]
fvc = sub.FVC.values
weeks = sub.Weeks.values
c = np.vstack([weeks, np.ones(len(weeks)) ] ).T
a, b = np.linalg.lstsq(c, fvc)[0]
A[p] = a
TAB[p] = get_tab(sub)
 P.append(p )
pred_f=final_reg_fatal_f.predict(X_test_fatal_f)
 print("RMSE is:",(mean_squared_error(y_test_fatal_f,pred_f)) **0.5 )<feature_engineering>
class IGenerator(Sequence):
BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']
def __init__(self, keys, a, tab, batch_size=32):
self.keys = [k for k in keys if k not in self.BAD_ID]
self.a = a
self.tab = tab
self.batch_size = batch_size
self.train_data = {}
for p in train.Patient.values:
self.train_data[p] = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 def __len__(self):
return 1000
def __getitem__(self, idx):
x = []
a, tab = [], []
keys = np.random.choice(self.keys, size = self.batch_size)
 for k in keys:
try:
i = np.random.choice(self.train_data[k], size=1)[0]
img = get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')
 x.append(img)
 a.append(self.a[k])
 tab.append(self.tab[k])
 except:
print(k, i)
 x,a,tab = np.array(x), np.array(a), np.array(tab)
 x = np.expand_dims(x, axis=-1)
 return [x, tab] , a
test["Province_State"].fillna("state", inplace = True)
 test["Country_Region"] = [country_name.replace("'","")for country_name in test["Country_Region"]]<categorify>
Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D,
LeakyReLU, Concatenate 
)
 def get_efficientnet(model, shape):
models_dict = {
'b0': efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False),
'b1': efn.EfficientNetB1(input_shape=shape,weights=None,include_top=False),
'b2': efn.EfficientNetB2(input_shape=shape,weights=None,include_top=False),
'b3': efn.EfficientNetB3(input_shape=shape,weights=None,include_top=False),
'b4': efn.EfficientNetB4(input_shape=shape,weights=None,include_top=False),
'b5': efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False),
'b6': efn.EfficientNetB6(input_shape=shape,weights=None,include_top=False),
'b7': efn.EfficientNetB7(input_shape=shape,weights=None,include_top=False)
 }
return models_dict[model]
def build_model(shape=(512, 512, 1), model_class=None):
inp = Input(shape=shape)
 base = get_efficientnet(model_class, shape)
 x = base(inp)
 x = GlobalAveragePooling2D()(x)
 inp2 = Input(shape=(4,))
x2 = tf.keras.layers.GaussianNoise(0.2 )(inp2)
 x = Concatenate()([x, x2])
 x = Dropout(0.5 )(x)
 x = Dense(1 )(x)
 model = Model([inp, inp2] , x)
 weights = [w for w in os.listdir('.. /input/osic-model-weights')if model_class in w][0]
model.load_weights('.. /input/osic-model-weights/' + weights)
 return model
model_classes = ['b5']
models = [build_model(shape=(512, 512, 1), model_class=m)for m in model_classes]
print('Number of models: ' + str(len(models)) )
test['ConfirmedCases']=list(map(int,predicted_case))
test['Fatalities']=list(map(int,predicted_fatal))<save_to_csv>
tr_p, vl_p = train_test_split(P,
shuffle=True,
train_size= 0.8 )
submission=test[['ForecastId','ConfirmedCases','Fatalities']]
submission=shuffle(submission)
 submission.to_csv("submission.csv",index=False )<save_to_csv>
def score(fvc_true, fvc_pred, sigma):
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric )
with open('/kaggle/input/covid4submissions/week4_ml_30_final.csv', 'rt')as infile:
with open('submission.csv', 'wt')as outfile:
writer = csv.writer(outfile)
 for line in csv.reader(infile):
writer.writerow(line )<import_modules>
subs = []
for model in models:
metric = []
for q in tqdm(range(1, 10)) :
m = []
for p in vl_p:
x = []
tab = []
if p in ['ID00011637202177653955184', 'ID00052637202186188008618']:
continue
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/{i}'))
tab.append(get_tab(train.loc[train.Patient == p, :]))
if len(x)< 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q / 10)
 percent_true = train.Percent.values[train.Patient == p]
fvc_true = train.FVC.values[train.Patient == p]
weeks_true = train.Weeks.values[train.Patient == p]
fvc = a *(weeks_true - weeks_true[0])+ fvc_true[0]
percent = percent_true[0] - a * abs(weeks_true - weeks_true[0])
 m.append(score(fvc_true, fvc, percent))
print(np.mean(m))
metric.append(np.mean(m))
q =(np.argmin(metric)+ 1)/ 10
sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 A_test, B_test, P_test,W, FVC= {}, {}, {},{},{}
STD, WEEK = {}, {}
for p in test.Patient.unique() :
x = []
tab = []
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/{i}'))
tab.append(get_tab(test.loc[test.Patient == p, :]))
if len(x)<= 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q)
 A_test[p] = a
B_test[p] = test.FVC.values[test.Patient == p] - a*test.Weeks.values[test.Patient == p]
P_test[p] = test.Percent.values[test.Patient == p]
WEEK[p] = test.Weeks.values[test.Patient == p]
for k in sub.Patient_Week.values:
p, w = k.split('_')
 w = int(w)
 fvc = A_test[p] * w + B_test[p]
sub.loc[sub.Patient_Week == k, 'FVC'] = fvc
sub.loc[sub.Patient_Week == k, 'Confidence'] =(
 P_test[p] - A_test[p] * abs(WEEK[p] - w)
)
 _sub = sub[["Patient_Week","FVC","Confidence"]].copy()
subs.append(_sub )
from collections import defaultdict
import collections
import csv
from datetime import datetime, timedelta
import glob
import logging
from math import *
import numpy as np
import optparse
import os
import time
import sys
import copy
from typing import Any, Dict, List, Optional, Tuple, Union
import xgboost as xgb
from catboost import Pool, CatBoostRegressor
from xml.etree import ElementTree
from scipy.optimize import curve_fit<define_variables>
N = len(subs)
 sub = subs[0].copy()
sub["FVC"] = 0
sub["Confidence"] = 0
for i in range(N):
sub["FVC"] += subs[0]["FVC"] *(1/N)
 sub["Confidence"] += subs[0]["Confidence"] *(1/N )
RUN = False
PROD = True
PROD_DAYS = 30
EVAL_DAYS = 7<load_from_csv>
sub[["Patient_Week","FVC","Confidence"]].to_csv("submission_img.csv", index=False )
def read_learn_datasets(path, days):
examples = []
for i in range(days):
examples.extend(read_dataset(path + '_' + str(i+1)+ '.csv'))
return examples
def get_pool_data(mode, data, prefix):
def get(e):
return e if e else 0.0
x, delta_y, abs_y, fit, prev = [], [], [], [], []
for e in data[prefix + 'examples']:
if mode == 'cases':
cur_prev, cur_fit, cur_y = e.features['F_prev_cases_1'], e.features['F_fit_20'], get(e.cases)
 else:
cur_prev, cur_fit, cur_y = e.features['F_prev_fatal_1'], e.features['F_fit_fatal_20'], get(e.fatalities)
 cur_y = math.log(1 + cur_y)
 abs_y.append(cur_y)
 delta_y.append(cur_y - cur_prev)
 x.append(list(e.features.values()))
fit.append(cur_fit)
 prev.append(cur_prev)
 data['names'] = list(e.features.keys())
 data[prefix + 'x'] = np.asarray(x)
 data[prefix + 'delta_y'] = np.asarray(delta_y)
 data[prefix + 'abs_y'] = np.asarray(abs_y)
 data[prefix + 'fit'] = fit
data[prefix + 'prev'] = prev
def read_everything(opts):
data = {}
data['train_examples'] = read_learn_datasets(opts.train_features_csv, opts.days)
 data['test_examples'] = read_learn_datasets(opts.test_features_csv, opts.days)
 get_pool_data(opts.mode, data, 'train_')
 get_pool_data(opts.mode, data, 'test_')
 return data
def get_pool(data, prefix, is_delta, is_catboost):
y_key = prefix + 'delta_y' if is_delta else prefix + 'abs_y'
if is_catboost:
return Pool(data[prefix + 'x'], data[y_key])
 return xgb.DMatrix(data[prefix + 'x'], label=data[y_key])
 def write_predictions(path, data, mode, prefix, predicted, is_delta, name):
with open('_'.join([path, name, mode])+ '.csv', 'w')as f:
writer = csv.writer(f, lineterminator='
', delimiter=',')
 writer.writerow(['key', 'state', 'country', 'date', 'y', 'prediction', 'fit20', 'prev'])
 examples, y, fit, prev = data[prefix + 'examples'], data[prefix + 'abs_y'], data[prefix + 'fit'], data[prefix + 'prev']
for i in range(len(examples)) :
e = examples[i]
pred = predicted[i]
if is_delta:
pred += prev[i]
writer.writerow([e.key, e.state, e.country, e.date, y[i], max(pred, 0), fit[i], prev[i]])
 def model_name(is_delta, is_linear, is_catboost):
prefix = 'delta' if is_delta else 'abs'
if is_linear: return prefix + '_linear'
if is_catboost: return prefix + '_catboost'
return prefix + '_trees'
def do_learn(data, opts, is_delta, is_linear, is_catboost, iterations):
train = get_pool(data, 'train_', is_delta, is_catboost)
 test = get_pool(data, 'test_', is_delta, is_catboost)
 if is_catboost:
return
else:
param = {'nthread' : 16, 'objective' : 'reg:squarederror', 'seed' : opts.seed, 'eta' : opts.eta}
if is_linear:
param['booster'] = 'gblinear'
param['alpha'] = 0.00001
else:
param['max_depth'] = opts.depth
param['subsample'] = 0.8
evallist = [(test, 'eval'),(train, 'train')] if opts.verbose else []
booster = xgb.train(param, train, 2 if opts.super_quick else iterations, evallist)
 train_predicted, test_predicted = booster.predict(train), booster.predict(test)
 write_predictions(opts.train_predicted_csv, data, opts.mode, 'train_', train_predicted, is_delta, model_name(is_delta, is_linear, is_catboost))
write_predictions(opts.test_predicted_csv, data, opts.mode, 'test_', test_predicted, is_delta, model_name(is_delta, is_linear, is_catboost))
def main_LearningStage(opts):
data = read_everything(opts)
 iterations = {}
iterations[('delta', 'trees', 'cases')] = 170
iterations[('delta', 'trees', 'fatal')] = 260
iterations[('abs', 'trees', 'cases')] = 210
iterations[('abs', 'trees', 'fatal')] = 260
iterations[('delta', 'linear', 'cases')] = 4
iterations[('delta', 'linear', 'fatal')] = 10
iterations[('abs', 'linear', 'cases')] = 40
iterations[('abs', 'linear', 'fatal')] = 10
for is_linear in [False, True]:
for is_delta in [True, False]:
key =('delta' if is_delta else 'abs', 'linear' if is_linear else 'trees', opts.mode)
 print('__'.join(['Key'] + list(key)))
 do_learn(data, opts, is_delta, is_linear, False, iterations[key] )<compute_test_metric>
img_sub = sub[["Patient_Week","FVC","Confidence"]].copy()
def get_final_score(dt, at, dl, al, weights):
prev = float(dt['prev'])
 dt, at, dl, al = map(lambda x: float(x['prediction']), [dt, at, dl, al])
 pred = 0
pred += weights['wdt'] * dt
pred += weights['wat'] * at
pred += weights['wdl'] * dl
pred += weights['wal'] * al
return max(pred, prev)
 def coeff() :
return [0.0, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.5, 0.75, 0.85, 0.9, 1.0]
def ordered_loc_data(loc_data):
cur = [[k, loc_data[k]] for k in loc_data.keys() ]
cur.sort(key=lambda x: x[0])
 return [x[1] for x in cur]
def get_for_country(weights, values):
error, n = 0.0, 0
for v in values:
sc, sf = v['scores_cases'], v['scores_fatal']
cases = sc[0] * weights['wdt'] + sc[1] * weights['wat'] + sc[2] * weights['wdl'] + sc[3] * weights['wal']
fatal = sf[0] * weights['wdt'] + sf[1] * weights['wat'] + sf[2] * weights['wdl'] + sf[3] * weights['wal']
n += 2
error +=(v['y_cases'] - cases)** 2
fatal +=(v['y_fatal'] - fatal)** 2
return error
def prepare_index(examples_delta_trees, examples_abs_trees, examples_delta_linear, examples_abs_linear, opts, weights, country_weights):
index = {}
for e_dt, e_at, e_dl, e_al in zip(examples_delta_trees, examples_abs_trees, examples_delta_linear, examples_abs_linear):
key = '__'.join(map(str, [e_dt['country'], e_dt['state']]))
if country_weights is None:
pred = get_final_score(e_dt, e_at, e_dl, e_al, weights)
 else:
pred = get_final_score(e_dt, e_at, e_dl, e_al, country_weights[key])
 date = e_dt['date']
if key not in index: index[key] = {}
loc_index = index[key]
if date not in loc_index: loc_index[date] = {}
dct = loc_index[date]
dct['pred_' + e_dt['mode']] = pred
dct['y_' + e_dt['mode']] = float(e_dt['y'])
 dct['obj_' + e_dt['mode']] = e_dt
dct['prev_' + e_dt['mode']] = float(e_dt['prev'])
 dct['scores_' + e_dt['mode']] = list(map(lambda x: float(x['prediction']), [e_dt, e_at, e_dl, e_al]))
for key in index.keys() :
index[key] = ordered_loc_data(index[key])
 return index
def apply_rules(loc_data, min_mortality, max_mortality):
prev_cases = []
prev_fatalities = []
for v in loc_data:
cases, fatalities = v['pred_cases'], v['pred_fatal']
if len(prev_cases)> 0 and cases < prev_cases[-1]:
cases = prev_cases[-1]
if len(prev_fatalities)> 0 and fatalities < prev_fatalities[-1]:
fatalities = prev_fatalities[-1]
if cases - fatalities > min_mortality:
fatalities = cases - min_mortality
if cases - fatalities < max_mortality and cases > 5:
fatalities = cases - max_mortality
v['final_cases'] = max(cases, v['prev_cases'])
 v['final_fatal'] = max(fatalities, v['prev_fatal'])
 prev_cases.append(v['final_cases'])
 prev_fatalities.append(v['final_fatal'])
 def copy_and_set_prediction(e, pred, lst):
tmp = copy.deepcopy(e)
 tmp['prediction'] = pred
lst.append(tmp)
 def calc_error(examples_delta_trees, examples_abs_trees, examples_delta_linear, examples_abs_linear, opts, weights, country_weights):
index = prepare_index(examples_delta_trees, examples_abs_trees, examples_delta_linear, examples_abs_linear, opts, weights, country_weights)
 for loc_data in index.values() :
apply_rules(loc_data, math.log(opts.min_mortality), math.log(opts.max_mortality))
error = 0
cases_list, fatalities_list = [], []
n = 0
for loc_data in index.values() :
for v in loc_data:
if not opts.only_fatal:
n += 1
error +=(v['final_cases'] - v['y_cases'])** 2
if not opts.only_cases:
n += 1
error +=(v['final_fatal'] - v['y_fatal'])** 2
copy_and_set_prediction(v['obj_cases'], v['final_cases'], cases_list)
 copy_and_set_prediction(v['obj_fatal'], v['final_fatal'], fatalities_list)
 return(error / n)** 0.5, cases_list, fatalities_list
def read_examples(pattern, mode):
result = []
with open('{}_{}.csv'.format(pattern, mode), 'r')as f:
reader = csv.DictReader(f, quotechar='"', delimiter=',')
 for row in reader:
row['mode'] = mode
result.append(row)
 return result
def read_examples_for_model(opts, model_name):
test_examples = []
test_examples.extend(read_examples(opts.predicted_pattern + model_name, "cases"))
test_examples.extend(read_examples(opts.predicted_pattern + model_name, "fatal"))
return test_examples
def write_final_predictions(path, lst):
with open(path, 'wt')as f:
w = csv.writer(f)
 w.writerow(lst[0].keys())
 for e in lst:
w.writerow([e[k] for k in e.keys() ])
 def print_error(error, opts, w, win):
s = 'Test error = {}, Win = {}.min_mortality={}, max_mortality={}, formula = {} * DT + {} * AT + {} * DL + {} * AL'
return s.format(error, win, opts.min_mortality, opts.max_mortality, w['wdt'], w['wat'], w['wdl'], w['wal'])
 def main_FinalizingStage(opts):
examples_delta_trees = read_examples_for_model(opts, '_delta_trees')
 examples_abs_trees = read_examples_for_model(opts, '_abs_trees')
 examples_delta_linear = read_examples_for_model(opts, '_delta_linear')
 examples_abs_linear = read_examples_for_model(opts, '_abs_linear')
 country_weights = None
weights = {'wdt' : opts.wdt, 'wat' : opts.wat, 'wdl' : opts.wdl, 'wal' : opts.wal}
prod_error, cases_list, fatalities_list = calc_error(examples_delta_trees, examples_abs_trees, examples_delta_linear, examples_abs_linear, opts, weights, country_weights)
 print(prod_error)
 write_final_predictions(opts.predicted_pattern + '_cases.csv', cases_list)
 write_final_predictions(opts.predicted_pattern + '_fatalities.csv', fatalities_list)
<feature_engineering>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )
def main_Week2Adjustment(opts):
data = {}
with open(opts.train, 'r')as f:
reader = csv.DictReader(f, quotechar='"', delimiter=',')
 for row in reader:
key = '__'.join([row['Province_State'], row['Country_Region']])
 if key not in data: data[key] = [-1, -1, -1, -1]
if row['Date'] == '2020-03-27':
data[key][0] = math.log(1 + float(row['ConfirmedCases']))
data[key][1] = math.log(1 + float(row['Fatalities']))
if row['Date'] == '2020-03-31':
data[key][2] = math.log(1 + float(row['ConfirmedCases']))
data[key][3] = math.log(1 + float(row['Fatalities']))
hack = {}
with open(opts.cases_predicted_csv, 'r')as f:
writer = csv.writer(open(opts.modified_cases, 'wt'))
reader = csv.DictReader(f, quotechar='"', delimiter=',')
 last_country = 'None'
writer.writerow(['key','state','country','date','prediction', 'fit20'])
 for row in reader:
key = '__'.join([row['state'], row['country']])
 val = data[key]
if last_country != key:
last_country = key
last_value = data[key][2]
delta =(val[2] - val[0])/ 4
delta = max(0, delta)
 else:
delta *= 0.925
pred = float(row['prediction'])
 at_most = last_value + delta
pred = min(pred, at_most)
 last_value = pred
hack['__'.join([row['state'], row['country'], row['date']])] = pred
writer.writerow([row['key'], row['state'], row['country'], row['date'], pred, 0])
 with open(opts.fatal_predicted_csv, 'r')as f:
writer = csv.writer(open(opts.modified_fatal, 'wt'))
reader = csv.DictReader(f, quotechar='"', delimiter=',')
 last_country = 'None'
writer.writerow(['key','state','country','date','prediction', 'fit20'])
 for row in reader:
key = '__'.join([row['state'], row['country']])
 val = data[key]
if last_country != key:
last_country = key
last_value = data[key][3]
delta =(val[2] - val[0])/ 4
delta = max(0, delta)
 else:
delta *= 0.93
pred = float(row['prediction'])
 at_most = last_value + delta
pred = min(pred, at_most)
 cases_value = hack['__'.join([row['state'], row['country'], row['date']])]
if pred < cases_value - opts.max_mortality_log:
pred = cases_value - opts.max_mortality_log
last_value = pred
writer.writerow([row['key'], row['state'], row['country'], row['date'], pred, 0] )<load_from_csv>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )
def read_one(path):
res = []
with open(path, 'r')as f:
for row in csv.DictReader(f, quotechar='"', delimiter=','):
res.append(row)
 return res
def main_Submission(opts):
submission = read_one(opts.submission)
 cases = read_one(opts.predicted_cases)
 fatal = read_one(opts.predicted_fatalities)
 cases_index = {}
for case in cases:
cases_index[case['state'] + '__' + case['country'] + '__' + case['date']] = math.exp(float(case['prediction'])) - 1
fatal_index = {}
for case in fatal:
fatal_index[case['state'] + '__' + case['country'] + '__' + case['date']] = math.exp(float(case['prediction'])) - 1
n, found = 0, 0
with open(opts.output, 'w')as f:
writer = csv.writer(f, lineterminator='
', delimiter=',')
 writer.writerow(['ForecastId', 'ConfirmedCases', 'Fatalities'])
 for rec in submission:
key = rec['Province_State'] + '__' + rec['Country_Region'] + '__' + rec['Date']
cases_answer, fatal_answer = 0, 0
n += 1
if key in cases_index:
found += 1
cases_answer, fatal_answer = cases_index[key], fatal_index[key]
writer.writerow([rec['ForecastId'], cases_answer, fatal_answer])
 print('N={}'.format(n))
print('Found={}'.format(found))
print('Share={}'.format(float(found)/ n))<load_from_csv>
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique())
class FeaturesOpts:
def __init__(self, num_days, predict_window, verbose, start_date, last_n_days = -1):
self.train_csv = '/kaggle/input/covid19-global-forecasting-week-2/train.csv'
self.test_csv = '/kaggle/input/covid19-global-forecasting-week-2/test.csv'
self.train_features_csv = 'train_features_{}.csv'.format(predict_window)
 self.test_features_csv = 'test_features_{}.csv'.format(predict_window)
 self.num_days = num_days
self.predict_window = predict_window
self.verbose = verbose
self.start_date = start_date
self.last_n_days = last_n_days
start = time.time()
if RUN:
if PROD:
for i in range(PROD_DAYS):
print('Day ' + str(i + 1))
main_FeaturesStage(FeaturesOpts(PROD_DAYS, i + 1, False, '2020-04-02'))
else:
for i in range(EVAL_DAYS):
print('Day ' + str(i + 1))
main_FeaturesStage(FeaturesOpts(EVAL_DAYS, i + 1, False, '2020-03-31'))
end = time.time()
print('Overall took {} minutes'.format(( end - start)/ 60))<define_variables>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
class LearningOpts:
def __init__(self, num_days, mode, super_quick = False):
self.train_features_csv = 'train_features'
self.test_features_csv = 'test_features'
self.train_predicted_csv = 'train_predicted'
self.test_predicted_csv = 'test_predicted'
self.mode = mode
self.days = num_days
self.verbose = True
self.eta = 0.03
self.seed = 0
self.depth = 6
self.super_quick = super_quick
start = time.time()
if RUN:
if PROD:
main_LearningStage(LearningOpts(PROD_DAYS, 'cases'))
main_LearningStage(LearningOpts(PROD_DAYS, 'fatal'))
else:
main_LearningStage(LearningOpts(EVAL_DAYS, 'cases', False))
main_LearningStage(LearningOpts(EVAL_DAYS, 'fatal'))
end = time.time()
print('Overall took {} minutes'.format(( end - start)/ 60))<choose_model_class>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
class FinalizingOpts:
def __init__(self):
self.predicted_pattern = 'test_predicted'
self.verbose = True
self.only_cases = False
self.only_fatal = False
self.wdt = 0.5
self.wat = 0.3
self.wdl = 0.1
self.wal = 0.1
self.min_mortality = 250.0
self.max_mortality = 1.0
self.optimize = False
if RUN:
main_FinalizingStage(FinalizingOpts() )<load_from_csv>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
class Week2Opts:
def __init__(self):
self.train = '/kaggle/input/covid19-global-forecasting-week-2/train.csv'
self.cases_predicted_csv = 'test_predicted_cases.csv'
self.fatal_predicted_csv = 'test_predicted_fatalities.csv'
self.modified_cases = 'modified_cases.csv'
self.modified_fatal = 'modified_fatal.csv'
self.max_mortality_log = 4.7
if RUN and PROD:
main_Week2Adjustment(Week2Opts() )<load_from_csv>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )
class SubmissionOpts:
def __init__(self, use_modified = True):
self.submission = '/kaggle/input/covid19-global-forecasting-week-2/test.csv'
self.predicted_cases = 'modified_cases.csv' if use_modified else 'test_predicted_cases.csv'
self.predicted_fatalities = 'modified_fatal.csv' if use_modified else 'test_predicted_fatalities.csv'
self.output = 'submission.csv'
self.verbose = True
if RUN and PROD:
main_Submission(SubmissionOpts(True))<import_modules>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
<load_from_csv>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
train = pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 submission = pd.read_csv(".. /input/covid19-global-forecasting-week-4/submission.csv")
<data_type_conversions>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
train['Province_State'].fillna(" ", inplace = True)
 test['Province_State'].fillna(" ",inplace = True )<drop_column>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
nh = z.shape[1]
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
train['Country_Region']=train['Country_Region']+' '+train['Province_State']
test['Country_Region']=test['Country_Region']+' '+test['Province_State']
train.drop(['Province_State'],axis = 1 , inplace = True)
 test.drop(['Province_State'],axis = 1 , inplace = True )<data_type_conversions>
NFOLD = 5
kf = KFold(n_splits=NFOLD )
split_data_train = train["Date"].str.split("-" ).to_list()
split_data_test = test["Date"].str.split("-" ).to_list()
train_date = pd.DataFrame(split_data_train, columns=["Year","Month","Date"])
 test_date = pd.DataFrame(split_data_test, columns=["Year","Month","Date"])
 del train_date["Year"]
del test_date["Year"]
train_date['Month']=train_date['Month'].astype(int)
 test_date['Month']=test_date['Month'].astype(int)
 train_date['Date']=train_date['Date'].astype(int)
 test_date['Date']=test_date['Date'].astype(int)
 del train["Date"]
del test["Date"]
train = pd.concat([train,train_date],axis=1)
 test = pd.concat([test,test_date],axis=1)
<drop_column>
%%time
cnt = 0
EPOCHS = 800
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
train_Id = train["Id"]
del train["Id"]
test_Id = test["ForecastId"]
del test["ForecastId"]<prepare_x_and_y>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
train_x_full = train[['Country_Region','Month','Date']].copy()
train_y_full = train[['ConfirmedCases','Fatalities']].copy()<categorify>
sub['FVC1'] = 1.*pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
subm.loc[~subm.FVC1.isnull() ].head(10 )
le = LabelEncoder()
train_x_full['Country_Region']=le.fit_transform(train_x_full['Country_Region'])
 test['Country_Region']=le.transform(test['Country_Region'] )<split>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
x_train ,x_valid, y_train , y_valid = train_test_split(train_x_full , train_y_full , train_size = 0.85 , test_size = 0.15 )<import_modules>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
from sklearn.metrics import mean_absolute_error
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import RandomForestRegressor
from xgboost import XGBRegressor<choose_model_class>
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission_regression.csv", index=False )
model = LinearRegression()
model.fit(x_train,y_train)
 preds = model.predict(x_valid)
 print(mean_absolute_error(preds,y_valid))<predict_on_test>
reg_sub = subm[["Patient_Week","FVC","Confidence"]].copy()
model = RandomForestRegressor(n_estimators = 1000)
 model.fit(x_train,y_train)
 preds = model.predict(x_valid)
 print(mean_absolute_error(preds,y_valid))<predict_on_test>
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True )
model = XGBRegressor(n_estimators = 500, max_depth = 35)
 CC = y_train.ConfirmedCases
fatalities = y_train.Fatalities
model.fit(x_train, CC)
 predCC = model.predict(x_valid)
 model.fit(x_train, fatalities)
 predfa = model.predict(x_valid)
 p = pd.DataFrame({'ConfirmedCases': predCC, 'Fatalities' : predfa})
 print(mean_absolute_error(y_valid, p))<save_to_csv>
df = df1[['Patient_Week']].copy()
df['FVC'] = 0.25*df1['FVC'] + 0.75*df2['FVC']
df['Confidence'] = 0.25*df1['Confidence'] + 0.75*df2['Confidence']
df.head()
<load_from_csv><EOS> 
df.to_csv('submission.csv', index=False )