Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<split>
!pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index
!pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index
X_train, X_valid, y_train_cc, y_valid_cc = train_test_split(X, y_cc, random_state=42)
 y_train_ft = y_ft[y_train_cc.index]
y_valid_ft = y_ft[y_valid_cc.index]<predict_on_test>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )
def preds_country(X_fit, X_pred, y_fit, n_estimators=5, learning_rate=0.1, n_jobs=4, predict_type='XGBR'):

 X_pred_copy=X_pred.copy()
X_pred_copy['preds']=[np.nan for i in range(X_pred_copy.shape[0])]
for country in X_fit['Country_Region'].unique() :
if country in X_pred['Country_Region'].unique() :
for state in X_fit[X_fit['Country_Region']==country]['Province_State'].unique() :
if state in X_pred[X_pred['Country_Region']==country]['Province_State'].unique() :
X_fit_country = X_fit[(X_fit['Country_Region']==country)&(X_fit['Province_State']==state)].copy().drop(['Country_Region', 'Province_State'], axis=1)
 X_pred_country = X_pred[(X_pred['Country_Region']==country)&(X_pred['Province_State']==state)].copy().drop(['Country_Region', 'Province_State'], axis=1)
 y_fit_country = y_fit[X_fit_country.index]
if predict_type=='XGBR':
model = XGBRegressor(n_estimators=n_estimators, learning_rate=learning_rate, n_jobs=n_jobs, random_state=42)
 elif predict_type=='RFR':
model = RandomForestRegressor(n_estimators=n_estimators, random_state=42)
 else:
raise NameError('Model not valid')
 model.fit(X_fit_country, y_fit_country)
 X_pred_country['preds'] = model.predict(X_pred_country)
 for col in X_pred_country.index:
if col in X_pred_copy.index.values:
X_pred_copy.loc[col]=X_pred_country.loc[col]['preds']
return X_pred_copy<predict_on_test>
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=config )
<compute_test_metric>
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv' )
<predict_on_test>
def get_tab(df):
vector = [(df.Age.values[0] - 30)/ 30]
if df.Sex.values[0] == 'male':
vector.append(0)
 else:
vector.append(1)
 if df.SmokingStatus.values[0] == 'Never smoked':
vector.extend([0,0])
 elif df.SmokingStatus.values[0] == 'Ex-smoker':
vector.extend([1,1])
 elif df.SmokingStatus.values[0] == 'Currently smokes':
vector.extend([0,1])
 else:
vector.extend([1,0])
 return np.array(vector )
test_preds_cc = preds_country(covid_train[['Province_State', 'Country_Region', 'Date_st', 'Date_cc']], covid_test[['Province_State', 'Country_Region', 'Date_st', 'Date_cc']], covid_train['ConfirmedCases'], n_estimators=100, predict_type='RFR')
 test_preds_ft = preds_country(covid_train[['Province_State', 'Country_Region', 'Date_st', 'Date_ft']], covid_test[['Province_State', 'Country_Region', 'Date_st', 'Date_ft']], covid_train['Fatalities'], n_estimators=50, predict_type='RFR' )<save_to_csv>
A = {}
TAB = {}
P = []
for i, p in tqdm(enumerate(train.Patient.unique())) :
sub = train.loc[train.Patient == p, :]
fvc = sub.FVC.values
weeks = sub.Weeks.values
c = np.vstack([weeks, np.ones(len(weeks)) ] ).T
a, b = np.linalg.lstsq(c, fvc)[0]
A[p] = a
TAB[p] = get_tab(sub)
 P.append(p )
submission = pd.DataFrame({'ForecastId': test_preds_cc.index,'ConfirmedCases':test_preds_cc['preds'].round() ,'Fatalities':test_preds_ft['preds'].round() })
 filename = 'submission.csv'
submission.to_csv(filename,index=False )<import_modules>
class IGenerator(Sequence):
BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']
def __init__(self, keys, a, tab, batch_size=32):
self.keys = [k for k in keys if k not in self.BAD_ID]
self.a = a
self.tab = tab
self.batch_size = batch_size
self.train_data = {}
for p in train.Patient.values:
self.train_data[p] = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 def __len__(self):
return 1000
def __getitem__(self, idx):
x = []
a, tab = [], []
keys = np.random.choice(self.keys, size = self.batch_size)
 for k in keys:
try:
i = np.random.choice(self.train_data[k], size=1)[0]
img = get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')
 x.append(img)
 a.append(self.a[k])
 tab.append(self.tab[k])
 except:
print(k, i)
 x,a,tab = np.array(x), np.array(a), np.array(tab)
 x = np.expand_dims(x, axis=-1)
 return [x, tab] , a
import numpy as np
import pandas as pd
import seaborn as sns
from sklearn.model_selection import train_test_split
from xgboost import XGBRegressor
from sklearn.multioutput import MultiOutputRegressor
from sklearn.impute import SimpleImputer<load_from_csv>
Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D,
LeakyReLU, Concatenate 
)
 def get_efficientnet(model, shape):
models_dict = {
'b0': efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False),
'b1': efn.EfficientNetB1(input_shape=shape,weights=None,include_top=False),
'b2': efn.EfficientNetB2(input_shape=shape,weights=None,include_top=False),
'b3': efn.EfficientNetB3(input_shape=shape,weights=None,include_top=False),
'b4': efn.EfficientNetB4(input_shape=shape,weights=None,include_top=False),
'b5': efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False),
'b6': efn.EfficientNetB6(input_shape=shape,weights=None,include_top=False),
'b7': efn.EfficientNetB7(input_shape=shape,weights=None,include_top=False)
 }
return models_dict[model]
def build_model(shape=(512, 512, 1), model_class=None):
inp = Input(shape=shape)
 base = get_efficientnet(model_class, shape)
 x = base(inp)
 x = GlobalAveragePooling2D()(x)
 inp2 = Input(shape=(4,))
x2 = tf.keras.layers.GaussianNoise(0.2 )(inp2)
 x = Concatenate()([x, x2])
 x = Dropout(0.5 )(x)
 x = Dense(1 )(x)
 model = Model([inp, inp2] , x)
 weights = [w for w in os.listdir('.. /input/osic-model-weights')if model_class in w][0]
model.load_weights('.. /input/effnet-b5-30epochs-1/effnet_30.h5')
 return model
model_classes = ['b5']
models = [build_model(shape=(512, 512, 1), model_class=m)for m in model_classes]
print('Number of models: ' + str(len(models)) )
train_data = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 test_data = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 country_data = pd.read_csv('/kaggle/input/country-data-population/country_data_population.csv')
 submission_csv = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/submission.csv' )<merge>
tr_p, vl_p = train_test_split(P,
shuffle=True,
train_size= 1 )
train_data = pd.merge(train_data,country_data, left_on=['Country_Region'],right_on=['Country'], how='left')
 train_data.drop(['Country','Lending Category','UrbanRuralDesignation','GeoRegion','Continent'], axis=1, inplace=True)
 test_data = pd.merge(test_data,country_data, left_on=['Country_Region'],right_on=['Country'], how='left')
 test_data.drop(['Country','Lending Category','UrbanRuralDesignation','GeoRegion','Continent'], axis=1, inplace=True)
 train_data['GeoSubregion'].fillna('', inplace=True)
 test_data['GeoSubregion'].fillna('', inplace=True)
 train_data['Income Group'].fillna('', inplace=True)
 test_data['Income Group'].fillna('', inplace=True)
 train_data['Province_State'].fillna('', inplace=True)
 test_data['Province_State'].fillna('', inplace=True)
<data_type_conversions>
def score(fvc_true, fvc_pred, sigma):
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric )
train_data['Date'] = pd.to_datetime(train_data['Date'], infer_datetime_format=True)
 test_data['Date'] = pd.to_datetime(test_data['Date'], infer_datetime_format=True )<data_type_conversions>
subs = []
for model in models:
q = 0.5
sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 A_test, B_test, P_test,W, FVC= {}, {}, {},{},{}
STD, WEEK = {}, {}
for p in test.Patient.unique() :
x = []
tab = []
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 1.1 and int(i[:-4])/ len(ldir)> -0.1:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/{i}'))
tab.append(get_tab(test.loc[test.Patient == p, :]))
if len(x)<= 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q)
 A_test[p] = a
B_test[p] = test.FVC.values[test.Patient == p] - a*test.Weeks.values[test.Patient == p]
P_test[p] = test.Percent.values[test.Patient == p]
WEEK[p] = test.Weeks.values[test.Patient == p]
for k in sub.Patient_Week.values:
p, w = k.split('_')
 w = int(w)
 fvc = A_test[p] * w + B_test[p]
sub.loc[sub.Patient_Week == k, 'FVC'] = fvc
sub.loc[sub.Patient_Week == k, 'Confidence'] =(
 P_test[p] - A_test[p] * abs(WEEK[p] - w)
)
 _sub = sub[["Patient_Week","FVC","Confidence"]].copy()
subs.append(_sub )
train_data.loc[:, 'Date'] = train_data.Date.dt.strftime('%y%m%d')
 train_data.loc[:, 'Date'] = train_data['Date'].astype(int)
 test_data.loc[:, 'Date'] = test_data.Date.dt.strftime('%y%m%d')
 test_data.loc[:, 'Date'] = test_data['Date'].astype(int )<merge>
N = len(subs)
 sub = subs[0].copy()
sub["FVC"] = 0
sub["Confidence"] = 0
for i in range(N):
sub["FVC"] += subs[0]["FVC"] *(1/N)
 sub["Confidence"] += subs[0]["Confidence"] *(1/N )
all_data = pd.merge(train_data, test_data ,how='outer', on=['Country_Region','Date','Province_State'])
 pd.set_option('display.max_rows', None)
 all_data.sort_values(['Country_Region','Province_State','Date'], ascending = True, inplace = True)
 all_data = all_data.reset_index(drop=True)
 all_data.head(500 )<define_variables>
sub[["Patient_Week","FVC","Confidence"]].to_csv("submission_img.csv", index=False )
all_data['days_since_confirmedcases'] = 0.0
all_data['ConfirmedCases'].fillna(0.0, inplace=True)
 count = 0
save_province_state = ""
save_country_region = ""
save_confirmedcases = 0.0
first_confirmedcase = 0
for index, row in all_data.iterrows() :
if save_province_state != row['Province_State'] or save_country_region != row['Country_Region']:
save_province_state = row['Province_State']
save_country_region = row['Country_Region']
first_confirmedcase = 0
save_confirmedcases = 0
if(save_confirmedcases == 0.0)and(row['ConfirmedCases'] > 0.0):
save_confirmedcases = row['ConfirmedCases']
first_confirmedcase = count
if first_confirmedcase > 0:
all_data['days_since_confirmedcases'][count] = count - first_confirmedcase
count += 1
all_data.head(200 )<merge>
img_sub = sub[["Patient_Week","FVC","Confidence"]].copy()
train_data = pd.merge(train_data,all_data[['Province_State','Country_Region','Date','days_since_confirmedcases']], left_on=['Province_State','Country_Region','Date'],right_on=['Province_State','Country_Region','Date'], how='left')
 test_data = pd.merge(test_data,all_data[['Province_State','Country_Region','Date','days_since_confirmedcases']], left_on=['Province_State','Country_Region','Date'],right_on=['Province_State','Country_Region','Date'], how='left')
<data_type_conversions>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )
convert_dict = {'Province_State': str}
train_data = train_data.astype(convert_dict)
 test_data = test_data.astype(convert_dict )<define_variables>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )
s =(train_data.dtypes == 'object')
 object_cols = list(s[s].index )<import_modules>
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique())
from sklearn.preprocessing import LabelEncoder<categorify>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
label_encoder1 = LabelEncoder()
label_encoder2 = LabelEncoder()
label_encoder3 = LabelEncoder()
label_encoder4 = LabelEncoder()
train_data['Province_State'] = label_encoder1.fit_transform(train_data['Province_State'])
 test_data['Province_State'] = label_encoder1.transform(test_data['Province_State'])
 train_data['Country_Region'] = label_encoder2.fit_transform(train_data['Country_Region'])
 test_data['Country_Region'] = label_encoder2.transform(test_data['Country_Region'])
 train_data['GeoSubregion'] = label_encoder3.fit_transform(train_data['GeoSubregion'])
 test_data['GeoSubregion'] = label_encoder3.transform(test_data['GeoSubregion'])
 train_data['Income Group'] = label_encoder4.fit_transform(train_data['Income Group'])
 test_data['Income Group'] = label_encoder4.transform(test_data['Income Group'])
<define_variables>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
Test_id = test_data.ForecastId<drop_column>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
train_data.drop(['Id'], axis=1, inplace=True)
 test_data.drop('ForecastId', axis=1, inplace=True )<count_missing_values>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )
missing_val_count_by_column =(train_data.isnull().sum())
 print(missing_val_count_by_column[missing_val_count_by_column>0] )<import_modules>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 FE += ['age','week','BASE']
from xgboost import XGBRegressor<prepare_x_and_y>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
X_train = train_data[['Province_State','Country_Region','GeoSubregion','Income Group','days_since_confirmedcases','Date']]
y_train = train_data[['ConfirmedCases', 'Fatalities']]
X_test = test_data[['Province_State','Country_Region','GeoSubregion','Income Group','days_since_confirmedcases','Date']] 
<prepare_x_and_y>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
y_train_confirm = y_train.ConfirmedCases
y_train_fatality = y_train.Fatalities<train_model>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
nh = z.shape[1]
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
model1 = XGBRegressor(n_estimators=40000)
 model1.fit(X_train, y_train_confirm)
 y_pred_confirm = model1.predict(X_test )<train_model>
NFOLD = 2
kf = KFold(n_splits=NFOLD )
model2 = XGBRegressor(n_estimators=20000)
 model2.fit(X_train,y_train_fatality)
 y_pred_fat = model2.predict(X_test )<save_to_csv>
%%time
cnt = 0
EPOCHS = 600
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
df_sub = pd.DataFrame()
df_sub['ForecastId'] = Test_id
df_sub['ConfirmedCases'] = y_pred_confirm
df_sub['Fatalities'] = y_pred_fat
df_sub.to_csv('submission.csv', index=False )<compute_test_metric>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 terms_to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
return(sum(terms_to_sum)*(1.0/len(y)))** 0.5
def fix_target(frame, key, target, new_target_name="target"):
corrections = 0
group_keys = frame[ key].values.tolist()
target = frame[target].values.tolist()
for i in range(1, len(group_keys)- 1):
previous_group = group_keys[i - 1]
current_group = group_keys[i]
previous_value = target[i - 1]
current_value = target[i]
if current_group == previous_group:
if current_value<previous_value:
current_value=previous_value
target[i] =current_value
target[i] =max(0,target[i])
 frame[new_target_name] = np.array(target)
 def rate(frame, key, target, new_target_name="rate"):
corrections = 0
group_keys = frame[ key].values.tolist()
target = frame[target].values.tolist()
rate=[1.0 for k in range(len(target)) ]
for i in range(1, len(group_keys)) :
previous_group = group_keys[i - 1]
current_group = group_keys[i]
previous_value = target[i - 1]
current_value = target[i]
if current_group == previous_group:
if previous_value!=0.0:
rate[i]=current_value/previous_value
rate[i] =max(1,rate[i])
 frame[new_target_name] = np.array(rate)
 def difference(frame, key, target, new_target_name="diff"):
corrections = 0
group_keys = frame[ key].values.tolist()
target = frame[target].values.tolist()
rate=[0 for k in range(len(target)) ]
for i in range(1, len(group_keys)) :
previous_group = group_keys[i - 1]
current_group = group_keys[i]
previous_value = target[i - 1]
current_value = target[i]
if current_group == previous_group:
rate[i]=np.log1p(max(0,current_value-previous_value))
rate[i] =max(0,rate[i])
 frame[new_target_name] = np.array(rate)
 def get_data_by_key(dataframe, key, key_value, fields=None):
mini_frame=dataframe[dataframe[key]==key_value]
if not fields is None:
mini_frame=mini_frame[fields].values
return mini_frame
directory="/kaggle/input/coviddata/covid19-global-forecasting-week-4/"
model_directory="/kaggle/input/coviddata/model-dirv5/model"
us_county="/kaggle/input/coviddata/covid19-us-county-jhu-data-demographics/covid_us_county"
geo_dir=None
extra_stable_columns=None
group_by_columns=None
if not group_by_columns is None and "continent" in group_by_columns:
assert not geo_dir is None
train=pd.read_csv(directory + "train.csv", parse_dates=["Date"] , engine="python")
 test=pd.read_csv(directory + "test.csv", parse_dates=["Date"], engine="python")
 train["key"]=train[["Province_State","Country_Region"]].apply(lambda row: str(row[0])+ "_" + str(row[1]),axis=1)
 test["key"]=test[["Province_State","Country_Region"]].apply(lambda row: str(row[0])+ "_" + str(row[1]),axis=1)
 if not geo_dir is None:
region_metadata=pd.read_csv(geo_dir+"region_metadata.csv")
 km=KMeans(n_clusters=30, init='k-means++', n_init=10, max_iter=300, tol=0.0001,
precompute_distances='auto', verbose=0, random_state=1234, copy_x=True, n_jobs=None, algorithm='auto')
 region_metadata["clusters"]=km.fit_predict(region_metadata[["lat","lon"]].values)
 train=pd.merge(train,region_metadata, how="left", left_on=["Province_State","Country_Region"], right_on=["Province_State","Country_Region"])
 test=pd.merge(test,region_metadata, how="left", left_on=["Province_State","Country_Region"], right_on=["Province_State","Country_Region"])
 group_names=None
if not group_by_columns is None and len(group_by_columns)>0:
group_names=[]
for group in group_by_columns:
groupss=train[["Date", group,target1,target2]]
grp=groupss.groupby(["Date", group], as_index=False ).sum()
grp.columns=["Date", group,group +"_" + target1,group +"_" + target2 ]
train=pd.merge(train,grp, how="left", left_on=["Date", group,group], right_on=["Date", group,group])
 for gr in [group +"_" + target1,group +"_" + target2]:
rate(train, key, gr, new_target_name="rate_" +gr)
 group_names+=["rate_" +gr]
train.to_csv(directory + "train_plus_groups.csv", index=False)
 max_train_date=train["Date"].max()
max_test_date=test["Date"].max()
horizon=(max_test_date-max_train_date ).days
print("horizon", int(horizon))
target1="ConfirmedCases"
target2="Fatalities"
key="key"
train_supplamanteary2=None
new_data_file2=[us_county]
if not new_data_file2 is None:
trains=pd.read_csv(directory + "train.csv", parse_dates=["Date"] , engine="python")
 tests=pd.read_csv(directory + "test.csv", parse_dates=["Date"], engine="python")
 trains["key"]=train[["Province_State","Country_Region"]].apply(lambda row: str(row[0])+ "_" + str(row[1]),axis=1)
 tests["key"]=test[["Province_State","Country_Region"]].apply(lambda row: str(row[0])+ "_" + str(row[1]),axis=1)
 train_supplamanteary2=None
for new_data_files in new_data_file2:
sumplementary_data=pd.read_csv(new_data_files +".csv" , parse_dates=["date"] , engine="python")
 sumplementary_data.columns=["fips","Province_State","Country_Region","lat","long","Date",
"ConfirmedCases","state_code","Fatalities"]
sumplementary_data["key"]=sumplementary_data[["Province_State","Country_Region"]].apply(lambda row: str(row[0])+ "_" + str(row[1]),axis=1)
 sumplementary_data=sumplementary_data[["Date","key","ConfirmedCases","Fatalities","Province_State","Country_Region"]]
sumplementary_data["Id"]=[k for k in range(999999999,sumplementary_data.shape[0]+999999999)]
sumplementary_data=sumplementary_data[sumplementary_data.Date>= '2020-02-20']
if train_supplamanteary2 is None:
train_supplamanteary2=pd.concat([trains.reset_index(drop=True), sumplementary_data.reset_index(drop=True)], axis=0 ).reset_index(drop=True)
 else :
train_supplamanteary2=pd.concat([train_supplamanteary2.reset_index(drop=True), sumplementary_data.reset_index(drop=True)], axis=0 ).reset_index(drop=True)
 print(train_supplamanteary2.head(5))
train_supplamanteary2 = train_supplamanteary2.sort_values(["Country_Region","key","Id"], ascending =(True, True, True))
fix_target(train_supplamanteary2, key, target1, new_target_name=target1)
 fix_target(train_supplamanteary2, key, target2, new_target_name=target2)
 rate(train_supplamanteary2, key, target1, new_target_name="rate_" +target1)
 rate(train_supplamanteary2, key, target2, new_target_name="rate_" +target2)
 difference(train_supplamanteary2, key, target1, new_target_name="diff_" +target1)
 difference(train_supplamanteary2, key, target2, new_target_name="diff_" +target2)
 unique_keys_sup2=train_supplamanteary2[key].unique()
print(len(unique_keys_sup2))
if not geo_dir is None:
region_metadata=pd.read_csv(geo_dir+"region_metadata.csv")
 km=KMeans(n_clusters=30, init='k-means++', n_init=10, max_iter=300, tol=0.0001,
precompute_distances='auto', verbose=0, random_state=1234, copy_x=True, n_jobs=None, algorithm='auto')
 region_metadata["clusters"]=km.fit_predict(region_metadata[["lat","lon"]].values)
 train_supplamanteary2=pd.merge(train_supplamanteary2,region_metadata, how="left", left_on=["Province_State","Country_Region"], right_on=["Province_State","Country_Region"])
 if not group_by_columns is None and len(group_by_columns)>0:
group_names=[]
for group in group_by_columns:
groupss=train_supplamanteary2[["Date", group,target1,target2]]
grp=groupss.groupby(["Date", group], as_index=False ).sum()
grp.columns=["Date", group,group +"_" + target1,group +"_" + target2 ]
train_supplamanteary2=pd.merge(train_supplamanteary2,grp, how="left", left_on=["Date", group,group], right_on=["Date", group,group])
 for gr in [group +"_" + target1,group +"_" + target2]:
rate(train_supplamanteary2, key, gr, new_target_name="rate_" +gr)
 group_names+=["rate_" +gr]<categorify>
sub['FVC1'] = 1.*pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
subm.loc[~subm.FVC1.isnull() ].head(10 )
def get_lags(rate_array, current_index, size=20):
lag_confirmed_rate=[-1 for k in range(size)]
for j in range(0, size):
if current_index-j>=0:
lag_confirmed_rate[j]=rate_array[current_index-j]
else :
break
return lag_confirmed_rate
def days_ago_thresold_hit(full_array, indx, thresold):
days_ago_confirmed_count_10=-1
if full_array[indx]>thresold:
for j in range(indx,-1,-1):
entered=False
if full_array[j]<=thresold:
days_ago_confirmed_count_10=abs(j-indx)
 entered=True
break
if entered==False:
days_ago_confirmed_count_10=100
return days_ago_confirmed_count_10
def ewma_vectorized(data, alpha):
sums=sum([(alpha**(k+1)) *data[k] for k in range(len(data)) ])
 counts=sum([(alpha**(k+1)) for k in range(len(data)) ])
 return sums/counts
def generate_ma_std_window(rate_array, current_index, size=20, window=3):
ma_rate_confirmed=[-1 for k in range(size)]
std_rate_confirmed=[-1 for k in range(size)]
for j in range(0, size):
if current_index-j>=0:
ma_rate_confirmed[j]=np.mean(rate_array[max(0,current_index-j-window+1):current_index-j+1])
 std_rate_confirmed[j]=np.std(rate_array[max(0,current_index-j-window+1):current_index-j+1])
 else :
break
return ma_rate_confirmed, std_rate_confirmed
def generate_ewma_window(rate_array, current_index, size=20, window=3, alpha=0.05):
ewma_rate_confirmed=[-1 for k in range(size)]
for j in range(0, size):
if current_index-j>=0:
ewma_rate_confirmed[j]=ewma_vectorized(rate_array[max(0,current_index-j-window+1):current_index-j+1, ], alpha)
 else :
break
return ewma_rate_confirmed
def get_target(rate_col, indx, horizon=33, average=3, use_hard_rule=False):
target_values=[-1 for k in range(horizon)]
cou=0
for j in range(indx+1, indx+1+horizon):
if j<len(rate_col):
if average==1:
target_values[cou]=rate_col[j]
else :
if use_hard_rule and j +average <=len(rate_col):
target_values[cou]=np.mean(rate_col[j:j +average])
 else :
target_values[cou]=np.mean(rate_col[j:min(len(rate_col),j +average)])
 cou+=1
else :
break
return target_values
def get_target_count(rate_col, indx, horizon=33, average=3, use_hard_rule=False):
target_values=[-1 for k in range(horizon)]
cou=0
for j in range(indx+1, indx+1+horizon):
if j<len(rate_col):
if average==1:
target_values[cou]=rate_col[j]
else :
if use_hard_rule and j +average <=len(rate_col):
target_values[cou]=np.mean(rate_col[j:j +average])
 else :
target_values[cou]=np.mean(rate_col[j:min(len(rate_col),j +average)])
 cou+=1
else :
break
return target_values
def dereive_features(frame, confirmed, fatalities, rate_confirmed, rate_fatalities, count_confirmed, count_fatalities ,
horizon ,size=20, windows=[3,7], days_back_confimed=[1,10,100], days_back_fatalities=[1,2,10],
extra_data=None, groups_data=None, windows_group=[3,7], size_group=20,
days_back_confimed_group=[1,10,100]):
targets=[]
if not extra_data is None:
assert len(extra_stable_columns)==extra_data.shape[1]
if not groups_data is None:
assert len(group_names)==groups_data.shape[1]
names=[]
for day in days_back_confimed:
names+=["days_ago_confirmed_count_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["ma" + str(window)+ "_count_confirmed" + str(k+1)for k in range(size)]
for day in days_back_fatalities:
names+=["days_ago_fatalitiescount_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["ma" + str(window)+ "_count_fatalities" + str(k+1)for k in range(size)]
names+=["confirmed_level"]
names+=["fatalities_level"]
if not extra_data is None:
names+=[k for k in extra_stable_columns]
if not groups_data is None:
for gg in range(groups_data.shape[1]):
for day in days_back_confimed_group:
names+=["days_ago_grooupcount_" + str(gg+1)+ "_" + str(day)]
for window in windows_group:
names+=["ma_group_" + str(gg+1)+ "_" + str(window)+ "_rate_" + str(k+1)for k in range(size_group)]
names+=["confirmed_plus" + str(k+1)for k in range(horizon)]
names+=["fatalities_plus" + str(k+1)for k in range(horizon)]
names+=["confirmed_count_plus" + str(k+1)for k in range(horizon)]
names+=["fatalities_count_plus" + str(k+1)for k in range(horizon)]
features=[]
for i in range(len(confirmed)) :
row_features=[]
lag_confirmed_rate=get_lags(rate_confirmed, i, size=size)
 for day in days_back_confimed:
days_ago_confirmed_count_10=days_ago_thresold_hit(confirmed, i, day)
 row_features+=[days_ago_confirmed_count_10]
for window in windows:
ma3_rate_confirmed,std3_rate_confirmed= generate_ma_std_window(rate_confirmed, i, size=size, window=window)
 row_features+= ma3_rate_confirmed
ma3_count_confirmed,std3_count_confirmed= generate_ma_std_window(count_confirmed, i, size=size, window=window)
 row_features+= ma3_count_confirmed
lag_fatalities_rate=get_lags(rate_fatalities, i, size=size)
 for day in days_back_fatalities:
days_ago_fatalitiescount_2=days_ago_thresold_hit(fatalities, i, day)
 row_features+=[days_ago_fatalitiescount_2]
for window in windows:
ma3_rate_fatalities,std3_rate_fatalities= generate_ma_std_window(rate_fatalities, i, size=size, window=window)
 row_features+= ma3_rate_fatalities
ma3_count_fatalities,std3_count_fatalities= generate_ma_std_window(count_fatalities, i, size=size, window=window)
 row_features+= ma3_count_fatalities
confirmed_level=0

 confirmed_level= confirmed[i]
row_features+=[confirmed_level]
fatalities_is_level=0

 fatalities_is_level= fatalities[i]
row_features+=[fatalities_is_level]
if not extra_data is None:
row_features+=extra_data[i].tolist()
if not groups_data is None:
for gg in range(groups_data.shape[1]):
this_group=groups_data[:,gg].tolist()
lag_group_rate=get_lags(this_group, i, size=size_group)
 for day in days_back_confimed_group:
days_ago_groupcount_2=days_ago_thresold_hit(this_group, i, day)
 row_features+=[days_ago_groupcount_2]
for window in windows_group:
ma3_rate_group,std3_rate_group= generate_ma_std_window(this_group, i, size=size_group, window=window)
 row_features+= ma3_rate_group
confirmed_plus=get_target(rate_confirmed, i, horizon=horizon)
 row_features+= confirmed_plus
fatalities_plus=get_target(rate_fatalities, i, horizon=horizon)
 row_features+= fatalities_plus
confirmed_plus=get_target(count_confirmed, i, horizon=horizon)
 row_features+= confirmed_plus
fatalities_plus=get_target(count_fatalities, i, horizon=horizon)
 row_features+= fatalities_plus
features.append(row_features)
 new_frame=pd.DataFrame(data=features, columns=names ).reset_index(drop=True)
 frame=frame.reset_index(drop=True)
 frame=pd.concat([frame, new_frame], axis=1)
 return frame
def feature_engineering_for_single_key(frame, group, key, horizon=33, size=20, windows=[3,7],
days_back_confimed=[1,10,100], days_back_fatalities=[1,2,10],
extra_stable_=None, group_nams=None,windows_group=[3,7],
size_group=20, days_back_confimed_group=[1,10,100]):
mini_frame=get_data_by_key(frame, group, key, fields=None)
 mini_frame_with_features=dereive_features(mini_frame, mini_frame["ConfirmedCases"].values,
mini_frame["Fatalities"].values, mini_frame["rate_ConfirmedCases"].values,
mini_frame["rate_Fatalities"].values, mini_frame["diff_ConfirmedCases"].values,
mini_frame["diff_Fatalities"].values,
horizon ,size=size, windows=windows,
days_back_confimed=days_back_confimed, days_back_fatalities=days_back_fatalities,
extra_data=mini_frame[extra_stable_].values if not extra_stable_ is None else None,
groups_data=mini_frame[group_nams].values if not group_nams is None else None,
windows_group=windows_group, size_group=size_group,
days_back_confimed_group=days_back_confimed_group)
 return mini_frame_with_features<define_search_space>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
sigma_mean = 60
if sigma_mean<sigma_mean:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
train_frame=[]
size=10
windows=[3]
days_back_confimed=[1,5,10,20,50,100,250,500,1000]
days_back_fatalities=[1,2,5,10,20,50]
size_group=10
windows_group=[3]
days_back_confimed_group=[1,10,100]
for unique_k in tqdm(unique_keys):
mini_frame=feature_engineering_for_single_key(train, key, unique_k, horizon=horizon, size=size,
windows=windows, days_back_confimed=days_back_confimed,
days_back_fatalities=days_back_fatalities,
extra_stable_=extra_stable_columns if extra_stable_columns is not None and len(extra_stable_columns)>0 else None,
group_nams=group_names,windows_group=windows_group,
size_group=size_group, days_back_confimed_group=days_back_confimed_group
).reset_index(drop=True)
 train_frame.append(mini_frame)
 train_frame = pd.concat(train_frame, axis=0 ).reset_index(drop=True)
 new_unique_keys=train_frame['key'].unique()
for kee in new_unique_keys:
if kee not in unique_keys:
print(kee , " is not there ")
 print("==================================================")
 train_frame_supplamanteary2=None
if not train_supplamanteary2 is None:
train_frame_supplamanteary2=[]
for unique_k in tqdm(unique_keys_sup2):
mini_frame_sup=feature_engineering_for_single_key(train_supplamanteary2, key, unique_k, horizon=horizon, size=size,
windows=windows, days_back_confimed=days_back_confimed,
days_back_fatalities=days_back_fatalities,
extra_stable_=extra_stable_columns if extra_stable_columns is not None and len(extra_stable_columns)>0 else None,
group_nams=group_names,windows_group=windows_group,
size_group=size_group, days_back_confimed_group=days_back_confimed_group ).reset_index(drop=True)
 train_frame_supplamanteary2.append(mini_frame_sup)
 train_frame_supplamanteary2 = pd.concat(train_frame_supplamanteary2, axis=0 ).reset_index(drop=True)
 new_unique_keys_sup2=train_frame_supplamanteary2['key'].unique() 
<train_model>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
def bagged_set_train(X_ts,y_cs,wts, seed, estimators,xtest, xt=None,yt=None, output_name=None):
baggedpred=np.array([ 0.0 for d in range(0, xtest.shape[0])])
 for n in range(0, estimators):
params = {'objective': 'rmse',
'metric': 'rmse',
'boosting': 'gbdt',
'learning_rate': 0.005,
'drop_rate':0.01,
'skip_drop':0.6,
'uniform_drop':True,
'verbose': -1,
'num_leaves': 40,
'bagging_fraction': 0.9,
'bagging_freq': 1,
'bagging_seed': seed + n,
'feature_fraction': 0.8,
'feature_fraction_seed': seed + n,
'min_data_in_leaf': 10,
'max_bin': 100,
'max_depth':20,
'reg_alpha':1,
'lambda_l2': 10,
'num_threads':6
}
d_train = lgb.Dataset(X_ts,y_cs, weight=wts, free_raw_data=False)
 if not type(yt)is type(None):
d_cv = lgb.Dataset(xt,yt, free_raw_data=False, reference=d_train)
 model = lgb.train(params,d_train,num_boost_round=500,
valid_sets=d_cv,
verbose_eval=50)
 else :
model = lgb.train(params,d_train,num_boost_round=500)
 preds=model.predict(xtest)
 baggedpred+=preds
if not output_name is None:
joblib.dump(( model), output_name)

baggedpred/= estimators
return baggedpred 
<predict_on_test>
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission_regression.csv", index=False )
def predict(xtest,input_name=None):
baggedpred=np.array([ 0.0 for d in range(0, xtest.shape[0])])
 model= joblib.load(input_name)
 preds=model.predict(xtest)
 baggedpred+=preds
return baggedpred<define_variables>
reg_sub = subm[["Patient_Week","FVC","Confidence"]].copy()
names=[]
for day in days_back_confimed:
names+=["days_ago_confirmed_count_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
for day in days_back_fatalities:
names+=["days_ago_fatalitiescount_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["confirmed_level"]
names+=["fatalities_level"]
if not extra_stable_columns is None and len(extra_stable_columns)>0:
names+=[k for k in extra_stable_columns]
if not group_names is None:
for gg in range(len(group_names)) :
for day in days_back_confimed_group:
names+=["days_ago_grooupcount_" + str(gg+1)+ "_" + str(day)]
for window in windows_group:
names+=["ma_group_" + str(gg+1)+ "_" + str(window)+ "_rate_" + str(k+1)for k in range(size_group)] 
<define_variables>
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True )
<categorify>
df = df1[['Patient_Week']].copy()
df['FVC'] =(0.45*df1['FVC'] + 0.55*df2['FVC'])
 df['Confidence'] =(0.45*df1['Confidence'] + 0.55*df2['Confidence'])
 df.head()
<define_variables><EOS> 
df.to_csv('submission.csv', index=False )