Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<prepare_x_and_y>
!pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index
!pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index
X_train = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = data[data.date_block_num < 33]['item_cnt_month']
X_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = data[data.date_block_num == 33]['item_cnt_month']
X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1 )<set_options>
warnings.filterwarnings("ignore")
 %matplotlib inline
init_notebook_mode(connected=True)
 Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D,
Add, Conv2D, AveragePooling2D, LeakyReLU, Concatenate , Lambda 
)
 pd.set_option("display.max_columns", 100)
 custom_colors = ['
sns.palplot(sns.color_palette(custom_colors))
del data
gc.collect() ;<train_model>
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=config )
ts = time.time()
model = XGBRegressor(
 max_depth=10,
n_estimators=1000,
min_child_weight=0.5,
colsample_bytree=0.8,
subsample=0.8,
eta=0.1,
tree_method='gpu_hist',
seed=42)
 model.fit(
 X_train,
Y_train,
eval_metric="rmse",
eval_set=[(X_train, Y_train),(X_valid, Y_valid)],
verbose=True,
early_stopping_rounds = 20)
 time.time() - ts<save_to_csv>
EPOCHS = 30
BATCH_SIZE = 8
NFOLD = 5
LR = 0.003
SAVE_BEST = True
MODEL_CLASS = 'b1'
path = '.. /input/osic-pulmonary-fibrosis-progression/'
Y_pred = model.predict(X_valid ).clip(0, 20)
 Y_test = model.predict(X_test ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
})
 submission.to_csv('xgb_submission.csv', index=False)
 pickle.dump(Y_pred, open('xgb_train.pickle', 'wb'))
pickle.dump(Y_test, open('xgb_test.pickle', 'wb'))<install_modules>
tr = pd.read_csv('.. /input/osic-pf-train-meta-image/train_df_with_img_feat.csv')
 ts = pd.read_csv(f"{path}/test.csv")
 ts['Where'] = 'test'
ts_patient_ids = ts.Patient.unique().tolist()
!pip install -i https://test.pypi.org/simple/ litemort==0.1.7 
<choose_model_class>
percentile_range_1 = np.linspace(0.3,0.6,11)
 percentile_range_2 = range(30,63, 3)
 percentile_range = [(round(x,2),y)for x,y in zip(percentile_range_1,percentile_range_2)]
percentile_range
isMORT = True
alg='MORT' if isMORT else 'XGB'
print(f"gradient boosting lib={alg}")
<load_from_csv>
image_data_df = pd.DataFrame(image_data)
 image_data_df.head()
pd.set_option('display.max_rows', 500)
 pd.set_option('display.max_columns', 100)
 some_rows = None
data_root = '.. /input/'
test = pd.read_csv(f'{data_root}/competitive-data-science-predict-future-sales/test.csv' ).set_index('ID')
 data = pd.read_pickle(f'{data_root}/predict-future-sales/data.pkl')
 if some_rows is not None:
nMost=data.shape[0]
random.seed(42)
 subset = random.sample(range(nMost), some_rows)
 data = data.iloc[subset, :].reset_index(drop=True)
 print('====== Some Samples...data={}'.format(data.shape))
data = data[[
'date_block_num',
'shop_id',
'item_id',
'item_cnt_month',
'city_code',
'item_category_id',
'type_code',
'subtype_code',
'item_cnt_month_lag_1',
'item_cnt_month_lag_2',
'item_cnt_month_lag_3',
'item_cnt_month_lag_6',
'item_cnt_month_lag_12',
'date_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_2',
'date_item_avg_item_cnt_lag_3',
'date_item_avg_item_cnt_lag_6',
'date_item_avg_item_cnt_lag_12',
'date_shop_avg_item_cnt_lag_1',
'date_shop_avg_item_cnt_lag_2',
'date_shop_avg_item_cnt_lag_3',
'date_shop_avg_item_cnt_lag_6',
'date_shop_avg_item_cnt_lag_12',
'date_cat_avg_item_cnt_lag_1',
'date_shop_cat_avg_item_cnt_lag_1',
'date_city_avg_item_cnt_lag_1',
'date_item_city_avg_item_cnt_lag_1',
'delta_price_lag',
'month',
'days',
'item_shop_last_sale',
'item_last_sale',
'item_shop_first_sale',
'item_first_sale',
]]
X_train = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = data[data.date_block_num < 33]['item_cnt_month']
X_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = data[data.date_block_num == 33]['item_cnt_month']
X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1)
 print(f"X_train={X_train.shape} Y_train={Y_train.shape}")
 print(f"X_valid={X_valid.shape} Y_valid={Y_valid.shape}")
 print(f"X_test={X_test.shape} ")
 del data
gc.collect() ;<init_hyperparams>
ts = ts.merge(image_data_df, left_on = ['Patient'], right_on = ['Patient'], how = 'left')
 df = pd.concat([tr,ts] )
params={'num_leaves': 550, 'n_estimators':1000,'early_stopping_rounds':20,
'feature_fraction': 1, 'bagging_fraction': 1,
'max_bin': 512,
'max_depth': 10,
'min_child_weight': 300,
'learning_rate': 0.1,
'objective': 'regression',
'boosting_type': 'gbdt',
'verbose': 1,
'metric': {'rmse'}
}
if isMORT:
print(f"Call LiteMORT...")
 t0=time.time()
model = LiteMORT(params ).fit(X_train,Y_train,eval_set=[(X_valid, Y_valid)])
 print(f"LiteMORT...... OK time={time.time() -t0:.4g} model={model}")
 else:
model = XGBRegressor(
 max_depth=8,
n_estimators=1000,
min_child_weight=300,
colsample_bytree=0.8,
subsample=0.8,
eta=0.3,
seed=42)
 model.fit(
 X_train,
Y_train,
eval_metric="rmse",
eval_set=[(X_train, Y_train),(X_valid, Y_valid)],
verbose=True,
early_stopping_rounds = 10)
 alg = 'xgboost'
Y_pred = model.predict(X_valid ).clip(0, 20)
 score = np.sqrt(mean_squared_error(Y_pred, Y_valid))
Y_test = model.predict(X_test ).clip(0, 20)
 print(f"score={score}")
 def plot_features(booster, figsize):
fig, ax = plt.subplots(1,1,figsize=figsize)
 return plot_importance(booster=booster, ax=ax)
 if not isMORT:
plot_features(model,(10, 14)) 
<save_to_csv>
variables = ['ApproxVol_30_60','Avg_NumTissuePixel_30_60','Avg_Tissue_30_60','Avg_Tissue_thickness_30_60','Avg_TissueByTotal_30_60','Avg_TissueByLung_30_60']
for var in variables:
median = df[var].quantile(q = 0.5, interpolation='linear')
 df.loc[df[var]==0,var] = median
median = df['Avg_TissueByLung_30_60'].quantile(q = 0.5, interpolation='linear')
 df.loc[df['Avg_TissueByLung_30_60']>=0.3,'Avg_TissueByLung_30_60'] = median
median = df['Avg_Tissue_thickness_30_60'].quantile(q = 0.5, interpolation='linear')
 df.loc[df['Avg_Tissue_thickness_30_60']>=20000,'Avg_Tissue_thickness_30_60'] = median
df["Avg_Tissue_30_60_Quartile"] = pd.qcut(df.Avg_Tissue_30_60, q = 4, labels = ['Q1','Q2','Q3','Q4'] )
submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
})
 path = f'submission.csv'
submission.to_csv(path, index=False)
 print(f"...... Save submit @{path}...... ")
<set_options>
l = ['SliceThickness','PixelSpacing','ApproxVol_30_60','Avg_NumTissuePixel_30_60','Avg_Tissue_30_60',
'Avg_Tissue_thickness_30_60','Avg_TissueByTotal_30_60','Avg_TissueByLung_30_60']
for var in l:
df[var] =(df[var] - df[var].min())/(df[var].max() - df[var].min() )
warnings.filterwarnings('ignore' )<define_variables>
quartile = ['Q1','Q2','Q3','Q4']
for q in quartile:
temp_df = train.loc[train.Avg_Tissue_30_60_Quartile==q,:]
patients = temp_df.Patient.unique()
fvc_diff_lst = []
print('Quartile: ', q)
 for patient in patients:
fvc_lst = temp_df.loc[temp_df.Patient == patient, 'FVC'].tolist()
fvc_diff = fvc_lst[-1]-fvc_lst[0]
fvc_diff_lst.append(fvc_diff)
 print('FVC Diff Mean: ', sum(fvc_diff_lst)/len(fvc_diff_lst))
print('FVC Diff Q1: ', np.quantile(fvc_diff_lst,.25))
print('FVC Diff Q2/Median: ', np.quantile(fvc_diff_lst,.5))
print('FVC Diff Q3: ', np.quantile(fvc_diff_lst,.75))
print('FVC Diff Q4: ', np.quantile(fvc_diff_lst,.90))
data_dir = '/kaggle/input/competitive-data-science-predict-future-sales'
os.listdir(data_dir )<load_from_csv>
A = {}
TAB = {}
P = []
for i, p in tqdm(enumerate(train.Patient.unique())) :
sub = train.loc[train.Patient == p, :]
fvc = sub.FVC.values
weeks = sub.Weeks.values
c = np.vstack([weeks, np.ones(len(weeks)) ] ).T
a, b = np.linalg.lstsq(c, fvc)[0]
A[p] = a
TAB[p] = get_tab(sub)
 P.append(p )
train = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 test = pd.read_csv('/kaggle/input/competitive-data-science-predict-future-sales/test.csv' ).set_index('ID')
 train.head()<load_from_csv>
class IGenerator(Sequence):
BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']
def __init__(self, keys, a, tab, batch_size=32):
self.keys = [k for k in keys if k not in self.BAD_ID]
self.a = a
self.tab = tab
self.batch_size = batch_size
self.train_data = {}
for p in train.Patient.values:
self.train_data[p] = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 def __len__(self):
return 1000
def __getitem__(self, idx):
x = []
a, tab = [], []
keys = np.random.choice(self.keys, size = self.batch_size)
 for k in keys:
try:
i = np.random.choice(self.train_data[k], size=1)[0]
img = get_img_1(f'.. /input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')
 x.append(img)
 a.append(self.a[k])
 tab.append(self.tab[k])
 except:
print(k, i)
 x,a,tab = np.array(x), np.array(a), np.array(tab)
 x = np.expand_dims(x, axis=-1)
 return [x, tab] , a
item_catg = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/item_categories.csv')
 item_catg.info()<load_from_csv>
Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D,
LeakyReLU, Concatenate 
)
 def get_efficientnet(model, shape):
models_dict = {
'b0': efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False),
'b1': efn.EfficientNetB1(input_shape=shape,weights=None,include_top=False),
'b2': efn.EfficientNetB2(input_shape=shape,weights=None,include_top=False),
'b3': efn.EfficientNetB3(input_shape=shape,weights=None,include_top=False),
'b4': efn.EfficientNetB4(input_shape=shape,weights=None,include_top=False),
'b5': efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False),
'b6': efn.EfficientNetB6(input_shape=shape,weights=None,include_top=False),
'b7': efn.EfficientNetB7(input_shape=shape,weights=None,include_top=False)
 }
return models_dict[model]
def build_model(shape=(512, 512, 1), model_class = None, fold=None):
inp = Input(shape=shape)
 base = get_efficientnet(model_class, shape)
 x = base(inp)
 x = GlobalAveragePooling2D()(x)
 inp2 = Input(shape=(6,))
x2 = tf.keras.layers.GaussianNoise(0.2 )(inp2)
 x = Concatenate()([x, x2])
 x = Dropout(0.5 )(x)
 x = Dense(1 )(x)
 model = Model([inp, inp2] , x)
 weights = [w for w in os.listdir('.. /input/lindecay-efficientnetb1-models')if str(fold)in w][0]
model.load_weights('.. /input/lindecay-efficientnetb1-models/' + weights)
 return model
items = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/items.csv')
 items.info()<load_from_csv>
bst_quantile = [0.8, 0.5, 0.1, 0.1, 0.1]
shops = pd.read_csv('.. /input/competitive-data-science-predict-future-sales//shops.csv')
 shops.head()<save_to_csv>
model_class = 'b1'
NFOLD = 5
kf = KFold(n_splits=NFOLD, random_state=42,shuffle=False)
 P = np.array(P)
 subs = []
folds_history = []
for fold,(tr_idx, val_idx)in enumerate(kf.split(P)) :
print('
print('
print('
print('Predicting...')
 model = build_model(shape=(512, 512, 1), model_class = model_class, fold = fold)
 q = bst_quantile[fold]
sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 A_test, B_test, P_test,W, FVC= {}, {}, {}, {}, {}
STD, WEEK = {}, {}
for p in test.Patient.unique() :
x = []
tab = []
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/')
 counter = 0
for i in ldir:
if int(i[:-4])/ len(ldir)< 0.7 and int(i[:-4])/ len(ldir)> 0.20:
x.append(get_img_1(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/{i}'))
tab.append(get_tab(test.loc[test.Patient == p, :]))
counter+=1
if counter == 20:
break
if len(x)<= 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q)
 A_test[p] = a
B_test[p] = test.FVC.values[test.Patient == p] - a*test.Weeks.values[test.Patient == p]
P_test[p] = test.Percent.values[test.Patient == p]
WEEK[p] = test.Weeks.values[test.Patient == p]
for k in sub.Patient_Week.values:
p, w = k.split('_')
 w = int(w)
 fvc = A_test[p] * w + B_test[p]
sub.loc[sub.Patient_Week == k, 'FVC'] = fvc
sub.loc[sub.Patient_Week == k, 'Confidence'] =(
 P_test[p] - A_test[p] * abs(WEEK[p] - w)
)
 _sub = sub[["Patient_Week","FVC","Confidence"]].copy()
subs.append(_sub)
 K.clear_session()
del model
gc.collect()
<remove_duplicates>
N = len(subs)
 sub = subs[0].copy()
sub["FVC"] = 0
sub["Confidence"] = 0
for i in range(N):
sub["FVC"] += subs[i]["FVC"] *(1/N)
 sub["Confidence"] += subs[i]["Confidence"] *(1/N )
median = train[(train.item_price>0)].item_price.median()
train.loc[train.item_price<0,'item_price'] = median
train.drop_duplicates(subset =['date','date_block_num','item_id','item_price','item_cnt_day'],keep='first',inplace=True )<feature_engineering>
sub[["Patient_Week","FVC","Confidence"]].to_csv("submission_img.csv", index=False )
train.loc[train.shop_id==0,'shop_id']=57
test.loc[test.shop_id ==0,'shop_id']=57
train.loc[train.shop_id==1,'shop_id']=58
test.loc[test.shop_id ==1,'shop_id']=58
train.loc[train.shop_id==10,'shop_id']=11
test.loc[test.shop_id ==10,'shop_id']=11<merge>
img_sub = sub[["Patient_Week","FVC","Confidence"]].copy()
train = train.merge(test[['shop_id']].drop_duplicates() ,how = 'inner' )<categorify>
tr = pd.read_csv(f"{path}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{path}/test.csv")
 tr_patient_ids = tr.Patient.unique().tolist()
ts_patient_ids = chunk.Patient.unique().tolist()
sub = pd.read_csv(f"{path}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )
shops['city']=shops.shop_name.str.split(' ' ).map(lambda x:x[0])
 shops.loc[shops.city=='!Якутск','city'] = 'Якутск'
shops['shop_city'] = LabelEncoder().fit_transform(shops.city)
 shops['names'] = shops['shop_name'].apply(lambda x:x.lower() ).str.replace('[^\w\s]','' ).str.replace('\d+','' ).str.strip()
shops['shop_new_name']=LabelEncoder().fit_transform(shops.names )<feature_engineering>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
meta_data = tr.append([chunk, sub] )
shops['type']= shops.names.apply(lambda x: 'МТРЦ' if 'МТРЦ' in x
else 'ТРЦ' if 'ТРЦ' in x
else 'ТЦ' if 'ТЦ' in x
else 'ТРК' if 'ТРК' in x
else 'ТК' if 'ТК' in x
else 'No_data')
 shops['shop_type'] = shops.type.map({'No_data':0,'МТРЦ' :1,'ТРЦ':2,
'ТЦ':3, 'ТРК':4,'ТК' :5})
 shops = shops[['shop_id','shop_city','shop_new_name','shop_type']]<categorify>
print(tr.shape, chunk.shape, sub.shape, meta_data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
meta_data.Patient.nunique() )
item_catg['split'] = item_catg.item_category_name.str.split('-')
 item_catg['type'] = item_catg['split'].map(lambda x:x[0].strip())
 item_catg['item_type'] = LabelEncoder().fit_transform(item_catg['type'])
 item_catg['subtype']=item_catg.split.map(lambda x:x[1].strip() if len(x)>1 else x[0].strip())
 item_catg['item_subtype'] = LabelEncoder().fit_transform(item_catg.subtype)
 item_catg = item_catg[['item_category_id','item_type','item_subtype']]<categorify>
meta_data['min_week'] = meta_data['Weeks']
meta_data.loc[meta_data.WHERE=='test','min_week'] = np.nan
items['name1'],items['name2']=items.item_name.str.split('[',1 ).str
items['name1'],items['name3']=items.item_name.str.split('(',1 ).str
items.name2=items.name2.str.replace('[A-Za-z0-9A-Яа-я+]',' ' ).str.lower()
items.name3=items.name3.str.replace('[A-Za-z0-9A-Яа-я+]',' ' ).str.lower()
items= items.fillna('0')
 items['name_1'] =LabelEncoder().fit_transform(items.name1)
 items['name_2'] =LabelEncoder().fit_transform(items.name2)
 items['name_3'] =LabelEncoder().fit_transform(items.name3)
 items = items[['item_id','item_category_id','name_1','name_2','name_3']]<data_type_conversions>
meta_data['min_week'] = meta_data['Weeks']
meta_data.loc[meta_data.WHERE=='test','min_week'] = np.nan
meta_data['min_week'] = meta_data.groupby('Patient')['min_week'].transform('min' )
matrix = []
cols = [ 'date_block_num', 'shop_id', 'item_id']
for i in train.date_block_num.unique() :
sales = train[train.date_block_num == i]
matrix.append(np.array(list(product([i],sales.shop_id.unique() ,sales.item_id.unique())) ,dtype = 'int16'))
matrix = pd.DataFrame(np.vstack(matrix),columns = cols)
 matrix.date_block_num = matrix.date_block_num.astype(np.int8)
 matrix.shop_id = matrix.shop_id.astype(np.int8)
 matrix.item_id = matrix.item_id.astype(np.int16)
 matrix.sort_values(cols,inplace = True )<groupby>
base = meta_data.loc[meta_data.Weeks == meta_data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True)
 meta_data = meta_data.merge(base, on='Patient', how='left')
 base = meta_data.loc[meta_data.Weeks == meta_data.min_week]
base = base[['Patient','Percent']].copy()
base.columns = ['Patient','min_Percent']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True)
 meta_data = meta_data.merge(base, on='Patient', how='left')
 meta_data['base_week'] = meta_data['Weeks'] - meta_data['min_week']
del base
group = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day':['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True )<merge>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in meta_data[col].unique() :
print(col, mod)
 FE.append(mod)
 meta_data[mod] =(meta_data[col] == mod ).astype(int)
matrix = pd.merge(matrix,group,on=cols,how='left')
 matrix['item_cnt_month'] = matrix.item_cnt_month.fillna(0 ).clip(0,20 ).astype(np.float16 )<concatenate>
image_data_df = pd.DataFrame(image_data)
 image_data_df.head()
test['date_block_num'] = 34
matrix = pd.concat([matrix,test],ignore_index=True,sort=False,keys=cols)
 matrix.fillna(0,inplace=True)
 gc.collect()<merge>
tr1 = pd.read_csv('.. /input/osic-pf-train-meta-image/train_df_with_img_feat.csv')
 cols = image_data_df.columns
tr1 = tr1[cols]
tr1 = tr1.drop_duplicates(keep = 'first')
 tr1.head()
matrix = pd.merge(matrix,shops,on=['shop_id'],how = 'left')
 matrix = pd.merge(matrix,items,on=['item_id'],how='left')
 matrix = pd.merge(matrix,item_catg,on = ['item_category_id'],how='left')
 del shops,items,item_catg
gc.collect()<merge>
image_data_df = pd.concat([tr1,image_data_df])
 image_data_df.head()
def lag_feature(df,lags,col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id',col+'_lag_'+ str(i)]
shifted['date_block_num'] += i
df = pd.merge(df,shifted,on=['date_block_num','shop_id','item_id'],how='left')
 return df
def add(list_names,list_num):
global matrix
str1 = list_names[0]
for i in range(1,len(list_names)) :
str1 = str1+'_and_'+list_names[i]
str1 = str1+'_avg_item_cnt'
group = matrix.groupby(list_names ).agg({'item_cnt_month': ['mean']})
 group.columns = [str1]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=list_names, how='left')
 matrix[str1] = matrix[str1].astype(np.float16)
 matrix = lag_feature(matrix, list_num, str1)
 matrix.drop([str1], axis=1, inplace=True )<drop_column>
variables = ['ApproxVol_30_60','Avg_NumTissuePixel_30_60','Avg_Tissue_30_60','Avg_Tissue_thickness_30_60','Avg_TissueByTotal_30_60','Avg_TissueByLung_30_60']
for var in variables:
median = image_data_df[var].quantile(q = 0.5, interpolation='linear')
 image_data_df.loc[image_data_df[var]==0,var] = median
median = image_data_df['Avg_TissueByLung_30_60'].quantile(q = 0.5, interpolation='linear')
 image_data_df.loc[image_data_df['Avg_TissueByLung_30_60']>=0.3,'Avg_TissueByLung_30_60'] = median
median = image_data_df['Avg_Tissue_thickness_30_60'].quantile(q = 0.5, interpolation='linear')
 image_data_df.loc[image_data_df['Avg_Tissue_thickness_30_60']>=20000,'Avg_Tissue_thickness_30_60'] = median
matrix = lag_feature(matrix,[1,2,3],'item_cnt_month')
 add(['date_block_num'],[1])
 add(['date_block_num','item_id'],[1,2,3])
 add(['date_block_num','shop_id'],[1,2,3])
 add(['date_block_num','shop_city'],[1])
 add(['date_block_num','shop_new_name'],[1])
 add(['date_block_num','shop_type'],[1])
 add(['date_block_num','item_category_id'],[1])
 add(['date_block_num','item_type'],[1])
 add(['date_block_num','item_subtype'],[1])
 add(['date_block_num','shop_id','item_id'],[1])
 add(['date_block_num','shop_id','name_1'],[1])
 add(['date_block_num','shop_id','name_2'],[1])
 add(['date_block_num','shop_id','name_3'],[1])
 add(['date_block_num','shop_id','item_category_id'],[1] )<merge>
final_data = meta_data.merge(image_data_df, left_on = ['Patient','WHERE'], right_on = ['Patient', 'Data'], how = 'left')
 print(final_data.shape)
 print(final_data.shape)
 final_data.head()
group = train.groupby(['item_id'] ).agg({'item_price':['mean']})
 group.columns = ['item_avg_item_price']
group.reset_index(inplace = True)
 matrix = pd.merge(matrix,group,on=['item_id'],how = 'left')
 matrix['item_avg_item_price'] = matrix['item_avg_item_price'].astype(np.float16)
 group = train.groupby(['date_block_num','item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['date_item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_price'] = matrix['date_item_avg_item_price'].astype(np.float16)
 lags = [1,2,3,4,5,6]
matrix = lag_feature(matrix, lags, 'date_item_avg_item_price')
 for i in lags:
matrix['delta_price_lag_'+str(i)] = \ 
(matrix['date_item_avg_item_price_lag_'+str(i)] - matrix['item_avg_item_price'])/ matrix['item_avg_item_price']
def select_trend(row):
for i in lags:
if row['delta_price_lag_'+str(i)]:
return row['delta_price_lag_'+str(i)]
return 0
matrix['delta_price_lag'] = matrix.apply(select_trend, axis=1)
 matrix['delta_price_lag'] = matrix['delta_price_lag'].astype(np.float16)
 matrix['delta_price_lag'].fillna(0, inplace=True)
 fetures_to_drop = ['item_avg_item_price', 'date_item_avg_item_price']
for i in lags:
fetures_to_drop += ['date_item_avg_item_price_lag_'+str(i)]
fetures_to_drop += ['delta_price_lag_'+str(i)]
matrix.drop(fetures_to_drop, axis=1, inplace=True )<data_type_conversions>
final_data['age'] =(final_data['Age'] - final_data['Age'].min())/(final_data['Age'].max() - final_data['Age'].min())
 final_data['BASE_FVC'] =(final_data['min_FVC'] - final_data['min_FVC'].min())/(final_data['min_FVC'].max() - final_data['min_FVC'].min())
 final_data['week'] =(final_data['base_week'] - final_data['base_week'].min())/(final_data['base_week'].max() - final_data['base_week'].min())
 final_data['BASE_percent'] =(final_data['min_Percent'] - final_data['min_Percent'].min())/(final_data['min_Percent'].max() - final_data['min_Percent'].min())
 l = ['SliceThickness','PixelSpacing','ApproxVol_30_60','Avg_NumTissuePixel_30_60','Avg_Tissue_30_60',
'Avg_Tissue_thickness_30_60','Avg_TissueByTotal_30_60','Avg_TissueByLung_30_60']
for var in l:
final_data[var] =(final_data[var] - final_data[var].min())/(final_data[var].max() - final_data[var].min())
 FE += ['age','week','BASE_FVC','BASE_percent']
FE.extend(l )
ts = time.time()
cache = {}
matrix['item_shop_last_sale'] = -1
matrix['item_shop_last_sale'] = matrix['item_shop_last_sale'].astype(np.int8)
 for idx, row in matrix.iterrows() :
key = str(row.item_id)+' '+str(row.shop_id)
 if key not in cache:
if row.item_cnt_month!=0:
cache[key] = row.date_block_num
else:
last_date_block_num = cache[key]
matrix.at[idx, 'item_shop_last_sale'] = row.date_block_num - last_date_block_num
cache[key] = row.date_block_num
cache = {}
matrix['item_last_sale'] = -1
matrix['item_last_sale'] = matrix['item_last_sale'].astype(np.int8)
 for idx, row in matrix.iterrows() :
key = row.item_id
if key not in cache:
if row.item_cnt_month!=0:
cache[key] = row.date_block_num
else:
last_date_block_num = cache[key]
if row.date_block_num>last_date_block_num:
matrix.at[idx, 'item_last_sale'] = row.date_block_num - last_date_block_num
cache[key] = row.date_block_num
time.time() - ts<categorify>
tr = final_data.loc[final_data.WHERE=='train']
chunk = final_data.loc[final_data.WHERE=='val']
sub = final_data.loc[final_data.WHERE=='test']
matrix['item_first_sale']=matrix['date_block_num'] - matrix.groupby('item_id')['date_block_num'].transform('min' )<correct_missing_values>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def laplace_log_likelihood(actual_fvc, predicted_fvc, confidence, return_values = False):

 sd_clipped = np.maximum(confidence, 70)
 delta = np.minimum(np.abs(actual_fvc - predicted_fvc), 1000)
 metric = - np.sqrt(2)* delta / sd_clipped - np.log(np.sqrt(2)* sd_clipped)
 if return_values:
return metric
else:
return np.mean(metric)
 def make_model(FE):
z = Input(( len(FE),), name="Patient")
 x = Dense(100, activation="relu", name="d1" )(z)
 x = Dense(100, activation="relu", name="d2" )(x)
 p1 = Dense(3, activation="linear", name="p1" )(x)
 p2 = Dense(3, activation="relu", name="p2" )(x)
 preds = Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.65), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
def fill_na(df):
for col in df.columns:
if('_lag_' in col)&(df[col].isnull().any()):
if 'item_cnt' in col:
df[col].fillna(0,inplace=True)
 return df
matrix = matrix[matrix.date_block_num > 3]
matrix = fill_na(matrix)
 matrix=downcast_dtypes(matrix )<data_type_conversions>
y = tr['FVC'].values.astype(float)
 X_train = tr[FE].values
X_test = sub[FE].values
train_preds = np.zeros(( X_train.shape[0], 3))
test_preds = np.zeros(( X_test.shape[0], 3))
matrix['year'] = matrix['date_block_num']/12+2013
matrix['month'] = matrix['date_block_num']%12
days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 matrix['days'] = matrix['month'].map(days ).astype(np.int8)
 matrix['year'] = matrix.year.astype(np.int16 )<merge>
%%time
FE = ['Male', 'Female', 'Ex-smoker', 'Never smoked', 'Currently smokes', 'age', 'week', 'BASE_FVC', 'BASE_percent']
image_features = ['SliceThickness','PixelSpacing','ApproxVol_30_60','Avg_NumTissuePixel_30_60','Avg_Tissue_30_60',
'Avg_TissueByTotal_30_60','Avg_TissueByLung_30_60']
FE1 = FE
FE2 = FE+['ApproxVol_30_60']
FE3 = FE+['Avg_Tissue_thickness_30_60']
FE4 = FE+['Avg_TissueByLung_30_60']
FE5 = FE+['ApproxVol_30_60','Avg_Tissue_thickness_30_60','Avg_TissueByLung_30_60']
model_cnt = 1
for FE in [FE1, FE2, FE3, FE4, FE5]:
print('Model Count: ', model_cnt, '
Features Used: ', FE)
 y = tr['FVC'].values.astype(float)
 X_train = tr[FE].values
X_test = sub[FE].values
globals() ['train_preds_{}'.format(model_cnt)] = np.zeros(( X_train.shape[0], 3))
globals() ['test_preds_{}'.format(model_cnt)] = np.zeros(( X_test.shape[0], 3))
NFOLD = 5
gkf = GroupKFold(n_splits=NFOLD)
 groups = tr['Patient'].values
cnt = 0
EPOCHS = 800
BATCH_SIZE=128
for tr_idx, val_idx in gkf.split(X_train, y, groups):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(FE)
 net.fit(X_train[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(X_train[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(X_train[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(X_train[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 globals() ['train_preds_{}'.format(model_cnt)][val_idx] = net.predict(X_train[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 globals() ['test_preds_{}'.format(model_cnt)] += net.predict(X_test, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
predicted_fvc = globals() ['train_preds_{}'.format(model_cnt)][:,1]
confidence = globals() ['train_preds_{}'.format(model_cnt)][:,2]-globals() ['train_preds_{}'.format(model_cnt)][:,0]
model_score = laplace_log_likelihood(actual_fvc = y, predicted_fvc = predicted_fvc, confidence = confidence,
return_values = False)
 print('Overall Score: ', model_score)
 model_cnt+=1
weekary = []
t = 2
count = 0
for w in range(3):
for i in [31,28,31,30,31,30,31,31,30,31,30,31]:
a = [0,0,0,0,0,0,0,count]
count += 1
for j in range(i):
a[t] += 1
if t ==6:
t = -1
t += 1
weekary.append(a)
 weekary = pd.DataFrame(np.vstack(weekary),columns = ['week0','week1','week2','week3','week4','week5','week6','date_block_num'])
 matrix = pd.merge(matrix,weekary,on=['date_block_num'],how='left' )<load_pretrained>
scores = []
for i in range(1,6):
globals() ['predicted_fvc_{}'.format(i)] = globals() ['train_preds_{}'.format(i)][:,1]
globals() ['confidence_{}'.format(i)] = globals() ['train_preds_{}'.format(i)][:,2]-globals() ['train_preds_{}'.format(i)][:,0]
for i1 in np.linspace(0,0.5,11):
for i2 in np.linspace(0,0.5,11):
for i3 in np.linspace(0,0.5,11):
for i4 in np.linspace(0,0.5,11):
for i5 in np.linspace(0,0.5,11):
if i1+i2+i3+i4+i5 == 1:
train_preds =(i1*train_preds_1+
i2*train_preds_2+
i3*train_preds_3+
i4*train_preds_4+
i5*train_preds_5)
 predicted_fvc = train_preds[:,1]
confidence = train_preds[:,2]-train_preds[:,0]
score = laplace_log_likelihood(actual_fvc = y, predicted_fvc = predicted_fvc, confidence = confidence, return_values = False)
 scores.append(( i1,i2,i3,i4,i5,score))
scores = sorted(scores, key = lambda x: x[-1])
 scores[-10:]
matrix.to_pickle('data2.pkl')
 del matrix
gc.collect()<load_pretrained>
best_weights = scores[-1]
train_preds =(best_weights[0]*train_preds_1+best_weights[1]*train_preds_2+best_weights[2]*train_preds_3+
best_weights[3]*train_preds_4+best_weights[4]*train_preds_5)
 test_preds =(best_weights[0]*test_preds_1+best_weights[1]*test_preds_2+best_weights[2]*test_preds_3+
best_weights[3]*test_preds_4+best_weights[4]*test_preds_5 )
data = pd.read_pickle('data2.pkl')
 data = data[[
'date_block_num',
'shop_id',
'item_id',
'item_cnt_month',
'shop_type',
'item_category_id',
'name_1',
'name_2',
'name_3',
'item_subtype',
'year',
'month',
'days',
'item_cnt_month_lag_1',
'item_cnt_month_lag_2',
'item_cnt_month_lag_3',
'date_block_num_avg_item_cnt_lag_1',
'date_block_num_and_item_id_avg_item_cnt_lag_1',
'date_block_num_and_item_id_avg_item_cnt_lag_2',
'date_block_num_and_item_id_avg_item_cnt_lag_3',
'date_block_num_and_shop_id_avg_item_cnt_lag_1',
'date_block_num_and_shop_id_avg_item_cnt_lag_2',
'date_block_num_and_shop_id_avg_item_cnt_lag_3',
'date_block_num_and_item_category_id_avg_item_cnt_lag_1',
'date_block_num_and_shop_id_and_item_category_id_avg_item_cnt_lag_1',
'date_block_num_and_shop_id_and_item_id_avg_item_cnt_lag_1',
'delta_price_lag',
'item_shop_last_sale',
'item_last_sale',
'item_first_sale'
]]
X_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = data[data.date_block_num == 33]['item_cnt_month']
X_zong = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_zong = data[data.date_block_num < 33]['item_cnt_month']
X_train, X_test, y_train, y_test = train_test_split(X_valid,Y_valid,test_size = 0.25)
 X_tests = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1)
 xx = pd.concat([X_zong,X_train])
 yy = pd.concat([Y_zong,y_train])
 del data,X_valid,Y_valid,X_train,y_train,X_zong,Y_zong
gc.collect()<categorify>
sigma_opt = mean_absolute_error(y, train_preds[:,1])
 sigma_uncertain = train_preds[:,2] - train_preds[:,0]
sigma_mean = np.mean(sigma_uncertain)
 print(sigma_opt, sigma_mean)
xx.to_pickle('xx.pkl')
 yy.to_pickle('yy.pkl')
 X_test.to_pickle('X_test.pkl')
 y_test.to_pickle('y_test.pkl' )<train_model>
sub['FVC1'] = test_preds[:, 1]
sub['Confidence1'] = test_preds[:,2] - test_preds[:,0]
submission = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
submission.loc[~submission.FVC1.isnull() ].head(10 )
ts = time.time()
model = XGBRegressor(
 max_depth=10,
n_estimators=100,
min_child_weight=0.5,
colsample_bytree=0.9,
subsample=0.8,
num_round = 10000,
nthread = 16,
eta=0.1,
seed=1)
 model.fit(xx, yy, eval_metric="rmse", eval_set=[(xx,yy),(X_test,y_test)], verbose=True, early_stopping_rounds = 10)
 time.time() - ts<predict_on_test>
submission.loc[~submission.FVC1.isnull() ,'FVC'] = submission.loc[~submission.FVC1.isnull() ,'FVC1']
if sigma_mean < 70:
submission['Confidence'] = sigma_opt
else:
submission.loc[~submission.FVC1.isnull() ,'Confidence'] = submission.loc[~submission.FVC1.isnull() ,'Confidence1']
Y_tests = model.predict(X_tests ).clip(0, 20 )<save_to_csv>
org_test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(org_test)) :
submission.loc[submission['Patient_Week']==org_test.Patient[i]+'_'+str(org_test.Weeks[i]), 'FVC'] = org_test.FVC[i]
submission.loc[submission['Patient_Week']==org_test.Patient[i]+'_'+str(org_test.Weeks[i]), 'Confidence'] = 70
test = pd.read_csv('/kaggle/input/competitive-data-science-predict-future-sales/test.csv' ).set_index('ID')
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_tests
})
 submission.to_csv('xgb_submission1.csv', index=False )<data_type_conversions>
submission[["Patient_Week","FVC","Confidence"]].to_csv("submission_meta.csv", index = False )
%%time
train.date = pd.to_datetime(train.date,format='%d.%m.%Y')
 train.date<groupby>
reg_sub = submission[["Patient_Week","FVC","Confidence"]].copy()
%%time
data = train.groupby([train.date.apply(lambda x:x.strftime('%Y-%m')) ,'item_id','shop_id'] ).sum().reset_index()
data.head()<prepare_output>
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True )
data = data[['date','item_id','shop_id','item_cnt_day']]
data.head()<prepare_output>
df = df1[['Patient_Week']].copy()
df['FVC'] = 0.4*df1['FVC'] + 0.6*df2['FVC']
df['Confidence'] = 0.4*df1['Confidence'] + 0.6*df2['Confidence']
df.head()
<merge><EOS> 
df.to_csv('submission.csv', index=False )