Kaggle competition: ASHRAE - Great Energy Predictor III
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )<categorify>
black_day = 10
outlier = False
rescale = False
debug=False
num_rounds = 200
clip0=False
folds = 3
use_ucf=False
ucf_clip=False
ucf_year = [2017, 2018]
predmode='all'
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )<merge>
import gc
import os
from pathlib import Path
import random
import sys
from tqdm import tqdm_notebook as tqdm
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from IPython.core.display import display, HTML
from sklearn import preprocessing
from sklearn.model_selection import KFold
from sklearn.preprocessing import LabelEncoder
import lightgbm as lgb
from sklearn.metrics import mean_squared_error
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base<data_type_conversions>
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )<feature_engineering>
zone_dict={0:4,1:0,2:7,3:4,4:7,5:0,6:4,7:4,8:4,9:5,10:7,11:4,12:0,13:5,14:4,15:4}
def set_local(df):
for sid, zone in zone_dict.items() :
sids = df.site_id == sid
df.loc[sids, 'timestamp'] = df[sids].timestamp - pd.offsets.Hour(zone )
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']<filter>
%%time
root = Path('.. /input/ashrae-feather-format-for-fast-loading')
 train_df = pd.read_feather(root/'train.feather')
 weather_train_df = pd.read_feather(root/'weather_train.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather')
 root_median = Path('.. /input/site-meter-median')
 df_groupNum_median = pd.read_pickle(root_median/'df_groupNum_median.pickle')
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data<compute_test_metric>
building_meta_df = building_meta_df.merge(train_df[['building_id','meter']].drop_duplicates() , on='building_id' )
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.65), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model<prepare_x_and_y>
building_meta_df['groupNum_train'] = building_meta_df['site_id'].astype('int')*10 + building_meta_df['meter'].astype('int' )
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
nh = z.shape[1]
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))<choose_model_class>
train_df = train_df [ train_df['building_id'] != 1099 ]
NFOLD = 5
kf = KFold(n_splits=NFOLD )<predict_on_test>
building_meta_df['floor_area'] = building_meta_df.square_feet / building_meta_df.floor_count
%%time
cnt = 0
EPOCHS = 1000
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD<compute_test_metric>
en_holidays = holidays.England()
ir_holidays = holidays.Ireland()
ca_holidays = holidays.Canada()
us_holidays = holidays.UnitedStates()
def add_holiyday(df_weather):
en_idx = df_weather.query('site_id == 1 or site_id == 5' ).index
ir_idx = df_weather.query('site_id == 12' ).index
ca_idx = df_weather.query('site_id == 7 or site_id == 11' ).index
us_idx = df_weather.query('site_id == 0 or site_id == 2 or site_id == 3 or site_id == 4 or site_id == 6 or site_id == 8 or site_id == 9 or site_id == 10 or site_id == 13 or site_id == 14 or site_id == 15' ).index
df_weather['IsHoliday'] = 0
df_weather.loc[en_idx, 'IsHoliday'] = df_weather.loc[en_idx, 'timestamp'].apply(lambda x: en_holidays.get(x, default=0))
df_weather.loc[ir_idx, 'IsHoliday'] = df_weather.loc[ir_idx, 'timestamp'].apply(lambda x: ir_holidays.get(x, default=0))
df_weather.loc[ca_idx, 'IsHoliday'] = df_weather.loc[ca_idx, 'timestamp'].apply(lambda x: ca_holidays.get(x, default=0))
df_weather.loc[us_idx, 'IsHoliday'] = df_weather.loc[us_idx, 'timestamp'].apply(lambda x: us_holidays.get(x, default=0))
holiday_idx = df_weather['IsHoliday'] != 0
df_weather.loc[holiday_idx, 'IsHoliday'] = 1
df_weather['IsHoliday'] = df_weather['IsHoliday'].astype(np.uint8 )
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )<prepare_output>
set_local(weather_train_df)
 add_holiyday(weather_train_df )
sub['FVC1'] = 1.*pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
subm.loc[~subm.FVC1.isnull() ].head(10 )<feature_engineering>
train_df_black = train_df.copy()
train_df_black = train_df_black.merge(building_meta_df, on=['building_id', 'meter'], how='left')
 train_df_black = train_df_black.merge(weather_train_df, on=['site_id', 'timestamp'], how='left')
 train_df_black['black_count']=0
for bid in train_df_black.building_id.unique() :
df = train_df_black[train_df_black.building_id==bid]
for meter in df.meter.unique() :
dfm = df[df.meter == meter]
b =(dfm.meter_reading == 0 ).astype(int)
 train_df_black.loc[(train_df_black.building_id==bid)&(train_df_black.meter == meter), 'black_count'] = b.groupby(( ~b.astype(bool)).cumsum() ).cumsum()
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']<load_from_csv>
train_df = train_df.merge(train_df_black[['timestamp','building_id','meter','black_count']], on=['timestamp','building_id','meter'])
 train_df = train_df[train_df['black_count'] < 24*black_day].drop('black_count', axis=1 )
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1<save_to_csv>
del train_df_black
gc.collect()
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission_regression.csv", index=False )<prepare_output>
train_df = train_df.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")' )
reg_sub = subm[["Patient_Week","FVC","Confidence"]].copy()<sort_values>
( train_df[train_df.building_id == 954].meter_reading==0 ).astype(int ).plot()
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True )<feature_engineering>
train_df = train_df.query('not(building_id == 954 & meter_reading == 0)')
 train_df = train_df.query('not(building_id == 1221 & meter_reading == 0)' )
df = df1[['Patient_Week']].copy()
df['FVC'] = FVC_weight*df1['FVC'] +(1-FVC_weight)*df2['FVC']
df['Confidence'] = Confidence_weight*df1['Confidence'] +(1-Confidence_weight)*df2['Confidence']
df.head()<save_to_csv>
funny_bids = [993, 1168, 904, 954, 778, 1021]
print('before', len(train_df))
if outlier:
train_df.loc[(train_df.building_id == 993)&(train_df.meter == 0)&(train_df.meter_reading > 30000), 'meter_reading'] = 31921
train_df.loc[(train_df.building_id == 993)&(train_df.meter == 1)&(train_df.meter_reading > 90000), 'meter_reading'] = 96545.5
train_df = train_df[(( train_df.building_id == 1168)&(train_df.meter == 0)&(train_df.meter_reading >10000)) == False]
train_df.loc[(train_df.building_id == 904)&(train_df.meter == 0)&(train_df.meter_reading >10000), 'meter_reading'] = 11306
train_df = train_df[(( train_df.building_id == 954)&(train_df.meter_reading >10000)) ==False]
if rescale:
train_df.loc[(train_df.building_id == 778)&(train_df.meter == 1), 'meter_reading'] = train_df.loc[(train_df.building_id == 778)&(train_df.meter == 1), 'meter_reading']/ 100
train_df.loc[(train_df.building_id == 1021)&(train_df.meter == 3), 'meter_reading'] = train_df.loc[(train_df.building_id == 1021)&(train_df.meter == 3), 'meter_reading']/ 1000
train_df = train_df.reset_index()
print('after', len(train_df))
gc.collect()
df.to_csv('submission.csv', index=False )<import_modules>
train_df.loc[(train_df.building_id.isin(site_0_bids)) &(train_df.meter==0), 'meter_reading'] = train_df[(train_df.building_id.isin(site_0_bids)) &(train_df.meter==0)]['meter_reading'] * 0.2931
import os
import cv2
import pydicom
import pandas as pd
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
import scipy
from tqdm.notebook import tqdm<load_from_csv>
train_df['date'] = train_df['timestamp'].dt.date
train_df['meter_reading_log1p'] = np.log1p(train_df['meter_reading'] )
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv')
 train.head()<train_model>
def preprocess(df):
df["hour"] = df["timestamp"].dt.hour
df["day"] = df["timestamp"].dt.day
df["weekend"] = df["timestamp"].dt.weekday
df["month"] = df["timestamp"].dt.month
df["dayofweek"] = df["timestamp"].dt.dayofweek
model_stats = ols('FVC ~ Weeks + Percent + Age + Sex + SmokingStatus',
data=train ).fit()
print(model_stats.summary() )<create_dataframe>
preprocess(train_df )
train_org = train.copy()<data_type_conversions>
if use_ucf:
train_df = train_df.sort_values('month')
 train_df = train_df.reset_index()
weeks_to_pred = np.array([i for i in range(-12, 134)] ).astype(np.float32 )<feature_engineering>
A_list = []
for i in list(A.values()):
if i > 0:
val = i -(np.max(list(A.values())) + 0.00000001)
 else:
val = i
A_list.append(val)
 x=0
for i in A.keys() :
A[i] = A_list[x]
x += 1<normalization>
weather_train_df.isna().sum()
def get_img(path):
d = pydicom.dcmread(path)
 return cv2.resize(( d.pixel_array - d.RescaleIntercept)/(d.RescaleSlope * 1000),(512, 512))<define_variables>
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )
class IGenerator(Sequence):
BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']
def __init__(self, keys, a, tab, batch_size=32):
self.keys = [k for k in keys if k not in self.BAD_ID]
self.a = a
self.tab = tab
self.batch_size = batch_size
self.train_data = {}
for p in train.Patient.unique() :
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 numb = [float(i[:-4])for i in ldir]
self.train_data[p] = [i for i in os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15]
def __len__(self):
return 1000
def __getitem__(self, idx):
x = []
a, tab = [], []
keys = np.random.choice(self.keys, size = self.batch_size)
 for k in keys:
try:
i = np.random.choice(self.train_data[k], size=1)[0]
img = get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')
 mask = cv2.resize(cv2.imread(f'.. /input/osic-pulmonary-fibrosis-progression-lungs-mask/mask_clear/mask_clear/{k}/{i[:-4]}.jpg', 0),(512, 512)) > 0
img[~mask] = 0
x.append(img)
 a.append(self.a[k])
 tab.append(self.tab[k])
 except:
print(k, i)
 x,a,tab = np.array(x), np.array(a), np.array(tab)
 x = np.expand_dims(x, axis=-1)
 return [x, tab] , a<compute_test_metric>
weather_train_df = weather_train_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D,
LeakyReLU, Concatenate 
)
 def get_exp(beta=0.025):
def exp_coef(x, beta=beta):
return 0.5 * K.tanh(x*beta)- beta
return exp_coef
def get_model(shape=(512, 512, 1)) :
def res_block(x, n_features):
_x = x
x = BatchNormalization()(x)
 x = LeakyReLU(0.05 )(x)
 x = Conv2D(n_features, kernel_size=(3, 3), strides=(1, 1), padding='same' )(x)
 x = Add()([_x, x])
 return x
inp = Input(shape=shape)
 x = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same' )(inp)
 x = BatchNormalization()(x)
 x = LeakyReLU(0.05 )(x)
 x = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same' )(x)
 x = BatchNormalization()(x)
 x = LeakyReLU(0.05 )(x)
 x = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(x)
 x = Conv2D(8, kernel_size=(3, 3), strides=(1, 1), padding='same' )(x)
 for _ in range(2):
x = res_block(x, 8)
 x = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(x)
 x = Conv2D(16, kernel_size=(3, 3), strides=(1, 1), padding='same' )(x)
 for _ in range(2):
x = res_block(x, 16)
 x = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(x)
 x = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same' )(x)
 for _ in range(3):
x = res_block(x, 32)
 x = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(x)
 x = Conv2D(64, kernel_size=(3, 3), strides=(1, 1), padding='same' )(x)
 for _ in range(3):
x = res_block(x, 64)
 x = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(x)
 x = Conv2D(128, kernel_size=(3, 3), strides=(1, 1), padding='same' )(x)
 for _ in range(3):
x = res_block(x, 128)
 x = GlobalAveragePooling2D()(x)
 inp2 = Input(shape=(4,))
x2 = tf.keras.layers.GaussianNoise(0.2 )(inp2)
 x = Concatenate()([x, x2])
 x = Dropout(0.5 )(x)
 x = Dense(1, activation='tanh' )(x)
 x = Dense(1, activation=get_exp(beta=0.025))(x)
 return Model([inp, inp2] , x )<compute_test_metric>
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )
def exp_coef(x, beta=1.0):
orig = x
return 0.5 * K.tanh(x*beta)- beta
a = [i / 10 for i in range(-10, 10)]
plt.plot(a, exp_coef(tf.convert_to_tensor(np.array(a ).astype(np.float32)) , beta=0.025), 'r.' )<choose_model_class>
def add_lag_feature(weather_df, window=3):
group_df = weather_df.groupby('site_id')
 cols = ['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction', 'wind_speed']
rolled = group_df[cols].rolling(window=window, min_periods=0)
 lag_mean = rolled.mean().reset_index().astype(np.float16)
 lag_max = rolled.max().reset_index().astype(np.float16)
 lag_min = rolled.min().reset_index().astype(np.float16)
 lag_std = rolled.std().reset_index().astype(np.float16)
 for col in cols:
weather_df[f'{col}_mean_lag{window}'] = lag_mean[col]
weather_df[f'{col}_max_lag{window}'] = lag_max[col]
weather_df[f'{col}_min_lag{window}'] = lag_min[col]
weather_df[f'{col}_std_lag{window}'] = lag_std[col]
model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.001), loss='mse', metrics=['mean_absolute_error'] )<split>
add_lag_feature(weather_train_df, window=3)
 add_lag_feature(weather_train_df, window=72 )
tr_p, vl_p = train_test_split(P,
shuffle=True,
train_size= 0.8 )<choose_model_class>
year_map = building_meta_df.year_built.value_counts()
building_meta_df['year_cnt'] = building_meta_df.year_built.map(year_map)
 bid_map = train_df.building_id.value_counts()
train_df['bid_cnt'] = train_df.building_id.map(bid_map )
er = tf.keras.callbacks.EarlyStopping(
 monitor="val_loss",
min_delta=1e-6,
patience=10,
verbose=0,
mode="auto",
baseline=None,
restore_best_weights=True,
)<train_model>
primary_use_list = building_meta_df['primary_use'].unique()
primary_use_dict = {key: value for value, key in enumerate(primary_use_list)}
print('primary_use_dict: ', primary_use_dict)
 building_meta_df['primary_use'] = building_meta_df['primary_use'].map(primary_use_dict)
 gc.collect()
model.fit_generator(IGenerator(keys=tr_p,
a = A,
tab = TAB),
steps_per_epoch = 100,
validation_data=IGenerator(keys=vl_p,
a = A,
tab = TAB),
validation_steps = 20,
callbacks = [er],
epochs=30 )<compute_test_metric>
train_df = reduce_mem_usage(train_df, use_float16=True)
 building_meta_df = reduce_mem_usage(building_meta_df, use_float16=True)
 weather_train_df = reduce_mem_usage(weather_train_df, use_float16=True )
def score(fvc_true, fvc_pred, sigma):
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric )<define_variables>
def add_sg(df):
w = 11
p = 2
for si in df.site_id.unique() :
index = df.site_id == si
df.loc[index, 'air_smooth'] = sg(df[index].air_temperature, w, p)
 df.loc[index, 'dew_smooth'] = sg(df[index].dew_temperature, w, p)
 df.loc[index, 'air_diff'] = sg(df[index].air_temperature, w, p, 1)
 df.loc[index, 'dew_diff'] = sg(df[index].dew_temperature, w, p, 1)
 df.loc[index, 'air_diff2'] = sg(df[index].air_temperature, w, p, 2)
 df.loc[index, 'dew_diff2'] = sg(df[index].dew_temperature, w, p, 2 )
metric = []
for q in tqdm(range(1, 10)) :
m = []
for p in vl_p:
x = []
tab = []
if p in ['ID00011637202177653955184', 'ID00052637202186188008618']:
continue
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/{i}'))
tab.append(get_tab(train.loc[train.Patient == p, :]))
if len(x)< 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = _a
a = np.quantile(_a, q / 10)
 percent_true = train.Percent.values[train.Patient == p]
fvc_true = train.FVC.values[train.Patient == p]
weeks_true = train.Weeks.values[train.Patient == p]
fvc = np.exp(a*(weeks_true - weeks_true[0])+ np.log(fvc_true[0]))
sigma = percent_true[0] - np.exp(abs(weeks_true - weeks_true[0])* a)
 m.append(score(fvc_true, fvc, sigma))
print(np.mean(m))
metric.append(np.mean(m))<define_variables>
add_sg(weather_train_df )
q =(np.argmin(metric)+ 1)/ 10
q<load_from_csv>
category_cols = ['building_id', 'site_id', 'primary_use', 'IsHoliday', 'groupNum_train']
feature_cols = ['square_feet', 'year_built'] + [
'hour', 'weekend',
] + [
'air_temperature', 'cloud_coverage',
'dew_temperature', 'precip_depth_1_hr',
'sea_level_pressure',
'air_temperature_mean_lag72',
'air_temperature_max_lag72', 'air_temperature_min_lag72',
'air_temperature_std_lag72', 'cloud_coverage_mean_lag72',
'dew_temperature_mean_lag72', 'precip_depth_1_hr_mean_lag72',
'sea_level_pressure_mean_lag72',
'wind_speed_mean_lag72',
'air_temperature_mean_lag3',
'air_temperature_max_lag3',
'air_temperature_min_lag3', 'cloud_coverage_mean_lag3',
'dew_temperature_mean_lag3',
'precip_depth_1_hr_mean_lag3',
'sea_level_pressure_mean_lag3',
'year_cnt', 'bid_cnt',
'dew_smooth', 'air_smooth',
'dew_diff', 'air_diff',
'dew_diff2', 'air_diff2'
] + list(df_groupNum_median.drop('timestamp',axis=1 ).columns )
sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 sub.head()<load_from_csv>
train_df = train_df.merge(building_meta_df, on=['building_id','meter'], how='left')
 train_df = train_df.merge(weather_train_df, on=['site_id', 'timestamp'], how='left')
 train_df = reduce_mem_usage(train_df, use_float16=True)
 del weather_train_df
gc.collect()
test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv' )<feature_engineering>
def create_X_y(train_df, groupNum_train):
target_train_df = train_df[train_df['groupNum_train'] == groupNum_train].copy()
target_train_df = target_train_df.merge(df_groupNum_median, on=['timestamp'], how='left')
 target_train_df['group_median_'+str(groupNum_train)] = np.nan
X_train = target_train_df[feature_cols + category_cols]
y_train = target_train_df['meter_reading_log1p'].values
del target_train_df
return X_train, y_train
A_test, B_test, P_test,W, FVC= {}, {}, {},{},{}
STD, WEEK = {}, {}
for p in test.Patient.unique() :
x = []
tab = []
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/{i}'))
tab.append(get_tab(test.loc[test.Patient == p, :]))
if len(x)<= 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = _a
a = np.quantile(_a, q)
 A_test[p] = a
B_test[p] = np.log(test.FVC.values[test.Patient == p])-(test.Weeks.values[test.Patient == p])* a
P_test[p] = test.Percent.values[test.Patient == p]
WEEK[p] = test.Weeks.values[test.Patient == p]<feature_engineering>
def fit_lgbm(train, val, devices=(-1,), seed=None, cat_features=None, num_rounds=1500, lr=0.1, bf=0.1):

 X_train, y_train = train
X_valid, y_valid = val
metric = 'l2'
params = {'num_leaves': 31,
'objective': 'regression',
'learning_rate': lr,
"boosting": "gbdt",
"bagging_freq": 5,
"bagging_fraction": bf,
"feature_fraction": 0.9,
"metric": metric,
}
device = devices[0]
if device == -1:
pass
else:
print(f'using gpu device_id {device}...')
 params.update({'device': 'gpu', 'gpu_device_id': device})
 params['seed'] = seed
early_stop = 20
verbose_eval = 50
d_train = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_features)
 d_valid = lgb.Dataset(X_valid, label=y_valid, categorical_feature=cat_features)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 y_pred_valid = model.predict(X_valid, num_iteration=model.best_iteration)
 print('best_score', model.best_score)
 log = {'train/mae': model.best_score['training']['l2'],
'valid/mae': model.best_score['valid_1']['l2']}
return model, y_pred_valid, log
for k in sub.Patient_Week.values:
p, w = k.split('_')
 w = int(w)
 fvc = np.exp(w*A_test[p] + B_test[p])
 sub.loc[sub.Patient_Week == k, 'FVC'] = fvc
sub.loc[sub.Patient_Week == k, 'Confidence'] = P_test[p] - np.exp(A_test[p] * abs(WEEK[p] - w)) 
<save_to_csv>
seed = 666
shuffle = False
kf = StratifiedKFold(n_splits=folds )
sub[["Patient_Week","FVC","Confidence"]].to_csv("submission_img.csv", index=False)
 img_sub = sub[["Patient_Week","FVC","Confidence"]].copy()<load_from_csv>
for groupNum_train in building_meta_df['groupNum_train'].unique() :
X_train, y_train = create_X_y(train_df, groupNum_train=groupNum_train)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('groupNum_train', groupNum_train, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 exec('models' +str(groupNum_train)+ '=[]')
 train_df_site = train_df[train_df['groupNum_train']==groupNum_train].copy()
for train_idx, valid_idx in kf.split(train_df_site, train_df_site['building_id']):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
mindex = train_df_site.iloc[valid_idx,:].month.unique()
print(mindex)
 print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,
num_rounds=num_rounds, lr=0.05, bf=0.7)
 y_valid_pred_total[valid_idx] = y_pred_valid
exec('models' +str(groupNum_train)+ '.append([mindex, model])')
 gc.collect()
if debug:
break
try:
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 plt.show()
except:
pass
del X_train, y_train
gc.collect()
print('-------------------------------------------------------------' )
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )<feature_engineering>
del train_df
gc.collect()
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub])
 data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )<merge>
print('loading...')
 test_df = pd.read_feather(root/'test.feather')
 weather_test_df = pd.read_feather(root/'weather_test.feather')
 weather_test_df = weather_test_df.drop_duplicates(['timestamp', 'site_id'])
 set_local(weather_test_df)
 add_holiyday(weather_test_df)
 print('preprocessing building...')
 test_df['date'] = test_df['timestamp'].dt.date
preprocess(test_df)
 print('preprocessing weather...')
 weather_test_df = weather_test_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
weather_test_df.groupby('site_id' ).apply(lambda group: group.isna().sum())
 add_sg(weather_test_df)
 add_lag_feature(weather_test_df, window=3)
 add_lag_feature(weather_test_df, window=72)
 test_df['bid_cnt'] = test_df.building_id.map(bid_map)
 test_df = test_df.merge(building_meta_df[['building_id','meter','groupNum_train']], on=['building_id','meter'], how='left')
 print('reduce mem usage...')
 test_df = reduce_mem_usage(test_df, use_float16=True)
 weather_test_df = reduce_mem_usage(weather_test_df, use_float16=True)
 gc.collect()
print(test_df.shape )
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True)
 data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base<data_type_conversions>
sample_submission = pd.read_feather(os.path.join(root, 'sample_submission.feather'))
reduce_mem_usage(sample_submission)
 print(sample_submission.shape )
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )<feature_engineering>
def create_X(test_df, groupNum_train):
target_test_df = test_df[test_df['groupNum_train'] == groupNum_train].copy()
target_test_df = target_test_df.merge(df_groupNum_median, on=['timestamp'], how='left')
 target_test_df = target_test_df.merge(building_meta_df, on=['building_id','meter','groupNum_train'], how='left')
 target_test_df = target_test_df.merge(weather_test_df, on=['site_id', 'timestamp'], how='left')
 target_test_df['group_median_'+str(groupNum_train)] = np.nan
X_test = target_test_df[feature_cols + category_cols]
return X_test
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']<filter>
def pred_all(X_test, models, batch_size=1000000):
iterations =(X_test.shape[0] + batch_size -1)// batch_size
print('iterations', iterations)
 y_test_pred_total = np.zeros(X_test.shape[0])
 for i,(mindex, model)in enumerate(models):
print(f'predicting {i}-th model')
 for k in tqdm(range(iterations)) :
y_pred_test = model.predict(X_test[k*batch_size:(k+1)*batch_size], num_iteration=model.best_iteration)
 y_test_pred_total[k*batch_size:(k+1)*batch_size] += y_pred_test
y_test_pred_total /= len(models)
 return y_test_pred_total
def pred(X_test, models, batch_size=1000000):
if predmode == 'valid':
print('valid pred')
 return pred_valid(X_test, models, batch_size=1000000)
 elif predmode == 'train':
print('train pred')
 return pred_train(X_test, models, batch_size=1000000)
 else:
print('all pred')
 return pred_all(X_test, models, batch_size=1000000 )
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data<compute_test_metric>
sample_submission.loc[(test_df.building_id.isin(site_0_bids)) &(test_df.meter==0), 'meter_reading'] = sample_submission[(test_df.building_id.isin(site_0_bids)) &(test_df.meter==0)]['meter_reading'] * 3.4118
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8375), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model<import_modules>
if clip0:
sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M<prepare_x_and_y>
if not debug:
sample_submission.to_csv('submission_no_leak.csv', index=False, float_format='%.4f' )
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
nh = z.shape[1]
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
NFOLD = 5
kf = KFold(n_splits=NFOLD )<predict_on_test>
leak_df = pd.read_feather('.. /input/ashrae-leak-data-station/leak.feather')
 print(leak_df.duplicated().sum())
 print(leak_df.meter.value_counts())
 leak_df.fillna(0, inplace=True)
 leak_df = leak_df[(leak_df.timestamp.dt.year > 2016)&(leak_df.timestamp.dt.year < 2019)]
leak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0
leak_df = leak_df[leak_df.building_id!=245]
sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0
test_df['pred'] = sample_submission.meter_reading
leak_df = leak_df.merge(test_df[['building_id', 'meter', 'timestamp', 'pred', 'row_id']], left_on = ['building_id', 'meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'], how = "left")
 leak_df = leak_df.merge(building_meta_df[['building_id', 'site_id']].drop_duplicates() , on='building_id', how='left' )
%%time
cnt = 0
EPOCHS = 800
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD<compute_test_metric>
leak_df['pred_l1p'] = np.log1p(leak_df.pred)
 leak_df['meter_reading_l1p'] = np.log1p(leak_df.meter_reading)
 sns.distplot(leak_df.pred_l1p)
 sns.distplot(leak_df.meter_reading_l1p)
 leak_score = np.sqrt(mean_squared_error(leak_df.pred_l1p, leak_df.meter_reading_l1p))
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )<prepare_output>
leak_df = leak_df[['meter_reading', 'row_id']].set_index('row_id' ).dropna()
sample_submission.loc[leak_df.index, 'meter_reading'] = leak_df['meter_reading']
sub['FVC1'] = 1.*pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
subm.loc[~subm.FVC1.isnull() ].head(10 )<feature_engineering>
if not debug:
sample_submission.to_csv('submission.csv', index=False, float_format='%.4f' )
<load_from_csv><EOS> 
print('LV score=', leak_score )