Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<sort_values>
%matplotlib inline
print(os.listdir(".. /input/house-prices-advanced-regression-techniques"))
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True )<feature_engineering>
train_data = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 test_data = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 sample_submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 data_desc = open(".. /input/house-prices-advanced-regression-techniques/data_description.txt")
 print(data_desc.read() )
df = df1[['Patient_Week']].copy()
df['FVC'] = 0.25*df1['FVC'] + 0.75*df2['FVC']
df['Confidence'] = 0.26*df1['Confidence'] + 0.74*df2['Confidence']
df.head()<save_to_csv>
data_null = train_data.isnull().sum() /len(train_data)* 100
data_null = data_null.drop(data_null[data_null == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio': data_null})
 missing_data.head(10 )
df.to_csv('submission.csv', index=False )<import_modules>
corr_data["SalePrice"].sort_values(ascending=False )
import tensorflow as tf
import pandas as pd
import numpy as np
import os
import random
import pathlib
import pickle
import time
import pydicom
import matplotlib.pyplot as plt
import tensorflow.keras.backend
from tensorflow import keras as K
from tensorflow.keras import layers as L
from skimage.transform import resize
from scipy.ndimage import zoom
import scipy.ndimage as ndimage
from skimage import measure, morphology, segmentation
from math import ceil<load_from_csv>
y_train = train_data["SalePrice"]
data = train_data.drop(["SalePrice"], axis=1 )
raw_test = pd.read_csv('/kaggle/input/osic-pulmonary-fibrosis-progression/test.csv')
 X_prediction = pd.read_csv('/kaggle/input/osic-pulmonary-fibrosis-progression/sample_submission.csv' )<define_variables>
train_data.sort_values(by = 'GrLivArea', ascending = False)[:2]
TEST_PATH = '/kaggle/input/osic-pulmonary-fibrosis-progression/test'
DESIRED_SIZE =(30,256,256)
 BATCH_SIZE = 256
MASK_ITERATION = 4
clip_bounds =(-1000, 200)
 pre_calculated_mean = 0.02865046213070556<create_dataframe>
train_data = train_data.drop(train_data[train_data['Id'] == 1299].index)
 train_data = train_data.drop(train_data[train_data['Id'] == 524].index )
<merge>
def find_outliers(model, X, y, sigma=3):
try:
y_pred = pd.Series(model.predict(X), index=y.index)
 except:
model.fit(X,y)
 y_pred = pd.Series(model.predict(X), index=y.index)
 resid = y - y_pred
mean_resid = resid.mean()
std_resid = resid.std()
z =(resid - mean_resid)/std_resid
outliers = z[abs(z)>sigma].index
return outliers
X_prediction['Patient'] = X_prediction['Patient_Week'].str.extract(r' (.*)_.*')
 X_prediction['Weeks'] = X_prediction['Patient_Week'].str.extract(r'.*_ (.*)' ).astype(int)
 X_prediction = X_prediction[['Patient', 'Weeks', 'Patient_Week']]
rename_cols = {'Weeks_y':'Min_week', 'Weeks_x': 'Weeks', 'FVC':'Base_FVC'}
X_prediction = X_prediction.merge(raw_test, how='left', left_on='Patient', right_on='Patient' ).rename(columns=rename_cols)[['Patient', 'Min_week', 'Base_FVC', 'Percent', 'Age', 'Sex', 'SmokingStatus', 'Weeks', 'Patient_Week']].reset_index(drop=True )<feature_engineering>
def changeSeqCat(data):
cols_ExGd = ['ExterQual','ExterCond','BsmtQual','BsmtCond',
'HeatingQC','KitchenQual','FireplaceQu','GarageQual',
'GarageCond','PoolQC']
dict_ExGd = {'Ex':5,'Gd':4,'TA':3,'Fa':2,'Po':1,'None':0}
for col in cols_ExGd:
data[col].replace(dict_ExGd, inplace=True)
 data['BsmtExposure'].replace({'Gd':4,'Av':3,'Mn':2,'No':1,'None':0}, inplace=True)
 data['CentralAir'].replace({'Y':1,'N':0}, inplace=True)
 data['Functional'].replace({'Typ':7,'Min1':6,'Min2':5,'Mod':4,'Maj1':3,'Maj2':2,'Sev':1,'Sal':0}, inplace=True)
 data['GarageFinish'].replace({'Fin':3,'RFn':2,'Unf':1,'None':0}, inplace=True)
 data['LotShape'].replace({'Reg':3,'IR1':2,'IR2':1,'IR3':0}, inplace=True)
 data['Utilities'].replace({'AllPub':3,'NoSewr':2,'NoSeWa':1,'ELO':0}, inplace=True)
 data['LandSlope'].replace({'Gtl':2,'Mod':1,'Sev':0}, inplace=True)
 return data
X_prediction['Base_week'] = X_prediction['Weeks'] - X_prediction['Min_week']<categorify>
def setMissingData(data):
data['MSZoning'] = data['MSZoning'].fillna(data['MSZoning'].mode() [0])
 data['LotFrontage'] = data['LotFrontage'].fillna(data['LotFrontage'].mean())
 data.OverallCond = data.OverallCond.astype(str)
 data['Fence'] = data['Fence'].fillna('NA')
 data['Alley'] = data['Alley'].fillna('NA')
 data['Functional'] = data['Functional'].fillna('Typ')
 data['MasVnrType'] = data['MasVnrType'].fillna(data['MasVnrType'].mode() [0])
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
data[col] = data[col].fillna('None')
 data['TotalBsmtSF'] = data['TotalBsmtSF'].fillna(0)
 data['KitchenQual'] = data['KitchenQual'].fillna(data['KitchenQual'].mode() [0])
 data['FireplaceQu'] = data['FireplaceQu'].fillna('None')
 for col in('GarageType', 'GarageFinish', 'GarageQual'):
data[col] = data[col].fillna('None')
 data['GarageCars'] = data['GarageCars'].fillna(0.0)
 data['SaleType'] = data['SaleType'].fillna(data['SaleType'].mode() [0])
 data['YrSold'] = data['YrSold'].astype(str)
 data['MoSold'] = data['MoSold'].astype(str)
 data['TotalSF'] = data['TotalBsmtSF'] + data['1stFlrSF'] + data['2ndFlrSF']
data['YrBltAndRemod']=data['YearBuilt']+data['YearRemodAdd']
data['Total_sqr_footage'] =(data['BsmtFinSF1'] + data['BsmtFinSF2'] +
data['1stFlrSF'] + data['2ndFlrSF'])
 data['Total_Bathrooms'] =(data['FullBath'] +(0.5 * data['HalfBath'])+
data['BsmtFullBath'] +(0.5 * data['BsmtHalfBath']))
data['Total_porch_sf'] =(data['OpenPorchSF'] + data['3SsnPorch'] +
data['EnclosedPorch'] + data['ScreenPorch'] +
data['WoodDeckSF'])
 data['haspool'] = data['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 data['has2ndfloor'] = data['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 data['hasgarage'] = data['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 data['hasbsmt'] = data['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 data['hasfireplace'] = data['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 return data
class OneHotEncoder(SklearnOneHotEncoder):
def __init__(self, **kwargs):
super(OneHotEncoder, self ).__init__(**kwargs)
 self.fit_flag = False
def fit(self, X, **kwargs):
out = super().fit(X)
 self.fit_flag = True
return out
def transform(self, X, categories, index='', name='', **kwargs):
sparse_matrix = super(OneHotEncoder, self ).transform(X)
 new_columns = self.get_new_columns(X=X, name=name, categories=categories)
 d_out = pd.DataFrame(sparse_matrix.toarray() , columns=new_columns, index=index)
 return d_out
def fit_transform(self, X, categories, index, name, **kwargs):
self.fit(X)
 return self.transform(X, categories=categories, index=index, name=name)
 def get_new_columns(self, X, name, categories):
new_columns = []
for j in range(len(categories)) :
new_columns.append('{}_{}'.format(name, categories[j]))
return new_columns
def standardisation(x, u, s):
return(x-u)/s
def normalization(x, ma, mi):
return(x-mi)/(ma-mi)
 class data_preparation() :
def __init__(self, bool_normalization=True,bool_standard=False):
self.enc_sex = LabelEncoder()
self.enc_smok = LabelEncoder()
self.onehotenc_smok = OneHotEncoder()
self.standardisation = bool_standard
self.normalization = bool_normalization
def __call__(self, data_untransformed):
data = data_untransformed.copy(deep=True)
 try:
data['Sex'] = self.enc_sex.transform(data['Sex'].values)
 data['SmokingStatus'] = self.enc_smok.transform(data['SmokingStatus'].values)
 data = pd.concat([data.drop(columns=['SmokingStatus']), self.onehotenc_smok.transform(data['SmokingStatus'].values.reshape(-1,1), categories=self.enc_smok.classes_, name='', index=data.index ).astype(int)], axis=1)
 if self.standardisation:
data['Base_week'] = standardisation(data['Base_week'],self.base_week_mean,self.base_week_std)
 data['Base_FVC'] = standardisation(data['Base_FVC'],self.base_fvc_mean,self.base_fvc_std)
 data['Base_percent'] = standardisation(data['Base_percent'],self.base_percent_mean,self.base_percent_std)
 data['Age'] = standardisation(data['Age'],self.age_mean,self.age_std)
 data['Weeks'] = standardisation(data['Weeks'],self.weeks_mean,self.weeks_std)
 if self.normalization:
data['Base_week'] = normalization(data['Base_week'],self.base_week_max,self.base_week_min)
 data['Base_FVC'] = normalization(data['Base_FVC'],self.base_fvc_max,self.base_fvc_min)
 data['Percent'] = normalization(data['Percent'],self.base_percent_max,self.base_percent_min)
 data['Age'] = normalization(data['Age'],self.age_max,self.age_min)
 data['Weeks'] = normalization(data['Weeks'],self.weeks_max,self.weeks_min)
 data['Min_week'] = normalization(data['Min_week'],self.base_week_max,self.base_week_min)
 except NotFittedError:
data['Sex'] = self.enc_sex.fit_transform(data['Sex'].values)
 data['SmokingStatus'] = self.enc_smok.fit_transform(data['SmokingStatus'].values)
 data = pd.concat([data.drop(columns=['SmokingStatus']), self.onehotenc_smok.fit_transform(data['SmokingStatus'].values.reshape(-1,1), categories=self.enc_smok.classes_, name='', index=data.index ).astype(int)], axis=1)
 if self.standardisation:
self.base_week_mean = data['Base_week'].mean()
self.base_week_std = data['Base_week'].std()
data['Base_week'] = standardisation(data['Base_week'],self.base_week_mean,self.base_week_std)
 self.base_fvc_mean = data['Base_FVC'].mean()
self.base_fvc_std = data['Base_FVC'].std()
data['Base_FVC'] = standardisation(data['Base_FVC'],self.base_fvc_mean,self.base_fvc_std)
 self.base_percent_mean = data['Base_percent'].mean()
self.base_percent_std = data['Base_percent'].std()
data['Base_percent'] = standardisation(data['Base_percent'],self.base_percent_mean,self.base_percent_std)
 self.age_mean = data['Age'].mean()
self.age_std = data['Age'].std()
data['Age'] = standardisation(data['Age'],self.age_mean,self.age_std)
 self.weeks_mean = data['Weeks'].mean()
self.weeks_std = data['Weeks'].std()
data['Weeks'] = standardisation(data['Weeks'],self.weeks_mean,self.weeks_std)
 if self.normalization:
self.base_week_min = data['Base_week'].min()
self.base_week_max = data['Base_week'].max()
data['Base_week'] = normalization(data['Base_week'],self.base_week_max,self.base_week_min)
 self.base_fvc_min = data['Base_FVC'].min()
self.base_fvc_max = data['Base_FVC'].max()
data['Base_FVC'] = normalization(data['Base_FVC'],self.base_fvc_max,self.base_fvc_min)
 self.base_percent_min = data['Percent'].min()
self.base_percent_max = data['Percent'].max()
data['Percent'] = normalization(data['Percent'],self.base_percent_max,self.base_percent_min)
 self.age_min = data['Age'].min()
self.age_max = data['Age'].max()
data['Age'] = normalization(data['Age'],self.age_max,self.age_min)
 self.weeks_min = data['Weeks'].min()
self.weeks_max = data['Weeks'].max()
data['Weeks'] = normalization(data['Weeks'],self.weeks_max,self.weeks_min)
 self.base_week_min = data['Min_week'].min()
self.base_week_max = data['Min_week'].max()
data['Min_week'] = normalization(data['Min_week'],self.base_week_max,self.base_week_min)
 return data<load_pretrained>
def handleSkew(all_data):
numeric_feats = all_data.dtypes[all_data.dtypes != "object"].index
skewed_feats = all_data[numeric_feats].apply(lambda x: skew(x.dropna())).sort_values(ascending=False)
 print("
Skew in numerical features:
")
 skewness = pd.DataFrame({'Skew' :skewed_feats})
 skewness.head(10)
 skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
all_data[feat] = boxcox1p(all_data[feat], lam)
 return all_data
pickefile = open('/kaggle/input/data-preparation-for-osic/data_prep', 'rb')
 data_prep = pickle.load(pickefile)
 pickefile.close()<sort_values>
class LabelBinarizerPipelineFriendly(MultiLabelBinarizer):
def fit(self, X, y=None):
super(LabelBinarizerPipelineFriendly,self ).fit(X)
 def transform(self, X, y=None):
return super(LabelBinarizerPipelineFriendly, self ).transform(X)
 def fit_transform(self, X, y=None):
return super(LabelBinarizerPipelineFriendly, self ).fit(X ).transform(X )
X_prediction = data_prep(X_prediction ).sort_values('Patient' )<data_type_conversions>
class DataFrameSelector(BaseEstimator, TransformerMixin):
def __init__(self, attr):
self.attributes = attr
def fit(self, X, y=None):
return self
def transform(self, X):
return X[self.attributes].values
class ConvertToHU:
def __call__(self, imgs, dicom):
intercept = dicom.RescaleIntercept
slope = dicom.RescaleSlope
imgs =(np.array(imgs.to_list())* slope + intercept ).astype(np.int16)
 return imgs
convertohu = ConvertToHU()<feature_engineering>
class LabelEncoderCat(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X = pd.DataFrame(X)
 X = X.apply(LabelEncoder().fit_transform)
 return X
class Clip:
def __init__(self, bounds=(-1000, 500)) :
self.min = min(bounds)
 self.max = max(bounds)
 def __call__(self, image):
image[image < self.min] = self.min
image[image > self.max] = self.max
return image
clip = Clip(clip_bounds )<define_search_space>
id_test = test_data['Id']
data = setMissingData(data)
 data = changeSeqCat(data)
 data = handleSkew(data )
class MaskWatershed:
def __init__(self, min_hu, iterations):
self.min_hu = min_hu
self.iterations = iterations
def __call__(self, image, dicom):
blackhat_struct = [[0, 0, 1, 1, 1, 0, 0],
[0, 1, 1, 1, 1, 1, 0],
[1, 1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1, 1],
[0, 1, 1, 1, 1, 1, 0],
[0, 0, 1, 1, 1, 0, 0]]
blackhat_struct = ndimage.iterate_structure(blackhat_struct, self.iterations)
 stack = []
for slice_idx in range(image.shape[0]):
sliced = image[slice_idx]
stack.append(self.seperate_lungs(sliced, blackhat_struct, self.min_hu,
self.iterations))
return np.stack(stack)
 @staticmethod
def seperate_lungs(image, blackhat_struct, min_hu=min(clip_bounds), iterations=2):
marker_internal, marker_external, marker_watershed = MaskWatershed.generate_markers(image)
 sobel_filtered_dx = ndimage.sobel(image, 1)
 sobel_filtered_dy = ndimage.sobel(image, 0)
 sobel_gradient = np.hypot(sobel_filtered_dx, sobel_filtered_dy)
 sobel_gradient *= 255.0 / np.max(sobel_gradient)
 watershed = morphology.watershed(sobel_gradient, marker_watershed)
 outline = ndimage.morphological_gradient(watershed, size=(3,3)).astype(bool)
 outline += ndimage.black_tophat(outline, structure=blackhat_struct)
 lungfilter = np.bitwise_or(marker_internal, outline)
 lungfilter = ndimage.morphology.binary_closing(lungfilter, structure=np.ones(( 5,5)) , iterations=3)
 segmented = np.where(lungfilter == 1, image, min_hu * np.ones(( image.shape[0], image.shape[1])))
 return segmented
@staticmethod
def generate_markers(image, threshold=-400):
marker_internal_labels = measure.label(segmentation.clear_border(image < threshold))
areas = [r.area for r in measure.regionprops(marker_internal_labels)]
areas.sort()
if len(areas)> 2:
for region in measure.regionprops(marker_internal_labels):
if region.area < areas[-2]:
for coordinates in region.coords:
marker_internal_labels[coordinates[0], coordinates[1]] = 0
marker_internal = marker_internal_labels > 0
external_a = ndimage.binary_dilation(marker_internal, iterations=10)
 external_b = ndimage.binary_dilation(marker_internal, iterations=55)
 marker_external = external_b ^ external_a
marker_watershed = np.zeros(( image.shape[0], image.shape[1]), dtype=np.int)
 marker_watershed += marker_internal * 255
marker_watershed += marker_external * 128
return marker_internal, marker_external, marker_watershed
maskwatershed = MaskWatershed(min_hu=min(clip_bounds), iterations=MASK_ITERATION )<normalization>
cat_attr = ['MSZoning', 'LandContour', 'Neighborhood', 'Condition1',
'BldgType', 'RoofMatl', 'Exterior1st', 'LotConfig', 'Alley',
'MasVnrType', 'BsmtFinType1', 'BsmtFinType2', 'Electrical',
'GarageType', 'SaleCondition', 'BsmtFinType1', 'RoofStyle']
num_attr = ['LotFrontage', 'LotArea', 'OverallQual', 'TotalSF', 'CentralAir',
'YearBuilt', 'BsmtFinSF1', 'BsmtUnfSF', 'BsmtExposure', 'GarageArea',
'GrLivArea', 'BsmtFullBath', 'FullBath', 'TotRmsAbvGrd', 'GarageCond',
'Fireplaces', 'WoodDeckSF','OpenPorchSF', 'GarageArea', 'GarageCars',
'haspool', 'has2ndfloor', 'hasgarage', 'hasbsmt', 'hasfireplace',
'YrBltAndRemod', 'Total_sqr_footage', 'Total_Bathrooms', 'Total_porch_sf',
'HeatingQC', 'KitchenQual', 'FireplaceQu', 'GarageQual', 'CentralAir',
'BsmtCond', 'BsmtQual', 'OverallCond', 'Functional', 'ExterQual']
num_pipeline = Pipeline([ 
('selector', DataFrameSelector(num_attr)) , 
('imputer', SimpleImputer(strategy="median")) , 
('scaler', StandardScaler()),
])
 cat_pipeline = Pipeline([ 
('selector', DataFrameSelector(cat_attr)) , 
('imputer', SimpleImputer(strategy="most_frequent")) , 
('label_binarizer', LabelBinarizerPipelineFriendly()),
])
 full_pipeline = FeatureUnion(transformer_list=[ 
('num_pipeline', num_pipeline), 
('cat_pipeline', cat_pipeline),
] )
class Normalize:
def __init__(self, bounds=(-1000, 500)) :
self.min = min(bounds)
 self.max = max(bounds)
 def __call__(self, image):
image = image.astype(np.float)
 image =(image - self.min)/(self.max - self.min)
 return image
class ZeroCenter:
def __init__(self, pre_calculated_mean):
self.pre_calculated_mean = pre_calculated_mean
def __call__(self, image):
return image - self.pre_calculated_mean
normalize = Normalize(bounds=clip_bounds)
 zerocenter = ZeroCenter(pre_calculated_mean=pre_calculated_mean )<sort_values>
n_folds = 5
def rmse_cv(model, X_data, y_data):
rmse= np.sqrt(-cross_val_score(model, X_data, y_data, scoring="neg_mean_squared_error", cv = 5))
return(rmse )
def sort_function(x):
return int(x.split('.')[0])
 def _read(path, patients=[], desired_size=(60,512,512)) :
X = np.empty(np.concatenate(( [len(patients), 1], np.array(desired_size))))
i=0
for patient in patients:
dicom = pydicom.dcmread(path+'/'+patient+'/'+os.listdir(path+'/'+patient)[0])
 list_patient_files = sorted([i for i in os.listdir(path+'/'+patient)], key=sort_function)
 df = pd.DataFrame(list_patient_files ).apply(lambda x:path+'/'+patient+'/'+x)
 df = convertohu(df.iloc[:,0].apply(lambda x: pydicom.dcmread(x ).pixel_array), dicom)
 df = zoom(df, np.array(DESIRED_SIZE)/np.array(df.shape), mode='nearest')
 X[i,0,:,:,:] = zerocenter(normalize(maskwatershed(clip(df), dicom)))
 i+=1
return X<define_variables>
X_prepared = pd.DataFrame(full_pipeline.fit_transform(data))
X_prepared = pd.DataFrame(X_prepared )
class DataGenerator(K.utils.Sequence):
def on_epoch_end(self):
self.indices = np.arange(len(self.list_IDs))
def __len__(self):
return int(ceil(len(self.indices)/ self.batch_size))
def __init__(self, train, list_IDs, batch_size=1, desired_size=(10,512,512), img_path=TEST_PATH, *args, **kwargs):
self.train = train
self.list_IDs = list_IDs
self.batch_size = batch_size
self.desired_size = desired_size
self.img_path = img_path
self.on_epoch_end()
def __getitem__(self, index):
indices = self.indices[index*self.batch_size:(index+1)*self.batch_size]
list_IDs_temp = [self.list_IDs[k] for k in indices]
patients = self.train.loc[list_IDs_temp, 'Patient'].unique()
imgs = _read(self.img_path, patients=patients, desired_size=self.desired_size)
 return self.train.loc[list_IDs_temp, :].reset_index(drop=True), np.transpose(imgs,(0, 2, 3, 4, 1))<create_dataframe>
data_null = data[cat_attr].isnull().sum() /len(data[cat_attr])* 100
data_null = data_null.drop(data_null[data_null == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio': data_null})
 missing_data
pred_generator = DataGenerator(X_prediction, X_prediction.index, batch_size=BATCH_SIZE, desired_size=DESIRED_SIZE, img_path=TEST_PATH )<load_pretrained>
final_data = pd.get_dummies(data[cat_attr] ).reset_index(drop=True)
 F, p_value = f_regression(final_data, y_train)
 np.array(final_data.columns)+ " = " +(p_value < 0.05 ).astype(str )
model = K.models.load_model('/kaggle/input/3d-cnn-mlp/model_4',compile=False)
 CNN = K.models.load_model('/kaggle/input/3d-cnn-mlp/CNN_4',compile=False )<categorify>
outliers = find_outliers(Ridge() , X_prepared, y_train)
 X_prepared = X_prepared.drop(outliers)
 y_train = y_train.drop(outliers)
 y_train = np.log1p(y_train )
SELECTED_COLUMNS = ['Weeks', 'Percent', 'Age', 'Sex', 'Min_week', 'Base_FVC','Base_week', '_Currently smokes', '_Ex-smoker', '_Never smoked']
y_prediction = np.array([[0,0,0]])
 for X1, X2 in pred_generator:
out_imgs = CNN(tf.convert_to_tensor(X2))
X_imgs = tf.concat([tf.repeat(tf.reshape(out_imgs[i],(1,-1)) , X1['Patient'].value_counts() [i], axis=0)for i in range(len(X1['Patient'].value_counts())) ], axis=0)
 X_tabular = model.get_layer('dense_to_freeze2' )(model.get_layer('dense_to_freeze1' )(tf.convert_to_tensor(np.asarray(X1[SELECTED_COLUMNS]))))
inp_mlp = model.get_layer('concatenate' )([X_imgs, X_tabular])
 inp_mlp = model.layers[-1](model.layers[-2](model.layers[-3](inp_mlp)))
 y_prediction = np.append(y_prediction, inp_mlp.numpy() , axis=0)
 y_prediction = y_prediction[1:] 
<save_to_csv>
lin_reg = LinearRegression()
score = rmse_cv(lin_reg, X_prepared, y_train)
 score.mean()
sub = pd.DataFrame(data={'Patient_Week':X_prediction['Patient_Week'], 'FVC':y_prediction[:,1], 'Confidence':(y_prediction[:,2]-y_prediction[:,0])})
 sub.to_csv('submission.csv', index=False )<install_modules>
ridge_reg = Ridge(alpha=0.001, solver="cholesky")
 score = rmse_cv(ridge_reg, X_prepared, y_train)
 score.mean()
!pip install /kaggle/input/kerasapplications -q
!pip install /kaggle/input/efficientnet-keras-source-code/ -q --no-deps<set_options>
lasso_reg = Lasso(alpha=0.0003)
 score = rmse_cv(lasso_reg, X_prepared, y_train)
 score.mean()
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
warnings.simplefilter('ignore' )<set_options>
elastic_net = ElasticNet(alpha=0.0005, l1_ratio=0.65)
 score = rmse_cv(elastic_net, X_prepared, y_train)
 score.mean()
MIXED_PRECISION = True
XLA_ACCELERATE = False
GPUS = tf.config.experimental.list_physical_devices('GPU')
 if GPUS:
try:
for GPU in GPUS:
tf.config.experimental.set_memory_growth(GPU, True)
 logical_gpus = tf.config.experimental.list_logical_devices('GPU')
 print(len(GPUS), "Physical GPUs,", len(logical_gpus), "Logical GPUs")
 except RuntimeError as RE:
print(RE)
 if MIXED_PRECISION:
policy = tf.keras.mixed_precision.experimental.Policy('mixed_float16')
 tf.keras.mixed_precision.experimental.set_policy(policy)
 print('Mixed precision enabled')
 if XLA_ACCELERATE:
tf.config.optimizer.set_jit(True)
 print('Accelerated Linear Algebra enabled')
 strategy = tf.distribute.get_strategy()
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}')
 print("Tensorflow version " + tf.__version__ )<define_variables>
ransac_reg = RANSACRegressor(lasso_reg)
 score = rmse_cv(ridge_reg, X_prepared, y_train)
 score.mean()
class BaseConfig(object):
SEED = 101
TRAIN_DF = '.. /input/ranzcr-clip-catheter-line-classification/train.csv'
TRAIN_IMG_PATH = '.. /input/ranzcr224/RANZCR_224/'
TEST_IMG_PATH = '.. /input/ranzcr-clip-catheter-line-classification/test/'
CLASS_MAP = '.. /input/ranzcr-clip-catheter-line-classification/train_annotations.csv'
SUBMIT = '.. /input/ranzcr-clip-catheter-line-classification/sample_submission.csv'<load_from_csv>
random_reg = RandomForestRegressor(max_depth=10, random_state=42, n_estimators=200)
 score = rmse_cv(random_reg, X_prepared, y_train)
 score.mean()
df = pd.read_csv(BaseConfig.TRAIN_DF)
 submit = pd.read_csv(BaseConfig.SUBMIT)
 target_cols = submit.columns[1:]
df.head()<define_variables>
huber_reg = HuberRegressor(epsilon=5)
 score = rmse_cv(huber_reg, X_prepared, y_train)
 score.mean()
target_cols = ['ETT - Abnormal', 'ETT - Borderline', 'ETT - Normal',
'NGT - Abnormal', 'NGT - Borderline', 'NGT - Incompletely Imaged',
'NGT - Normal', 'CVC - Abnormal', 'CVC - Borderline', 'CVC - Normal',
'Swan Ganz Catheter Present']<categorify>
svm_reg = LinearSVR(epsilon=0.001)
 score = rmse_cv(svm_reg, X_prepared, y_train)
 score.mean()
def build_decoder(with_labels=True, target_size=(300, 300), ext='jpg'):
def decode(path):
file_bytes = tf.io.read_file(path)
 if ext == 'png':
img = tf.image.decode_png(file_bytes, channels=3)
 elif ext in ['jpg', 'jpeg']:
img = tf.image.decode_jpeg(file_bytes, channels=3)
 else:
raise ValueError("Image extension not supported")
 img = tf.cast(img, tf.float32)/ 255.0
img = tf.image.resize(img, target_size)
 return img
def decode_with_labels(path, label):
return decode(path), label
return decode_with_labels if with_labels else decode
def build_augmenter(with_labels=True):
def augment(img):
img = tf.image.random_flip_left_right(img)
 img = tf.image.random_flip_up_down(img)
 return img
def augment_with_labels(img, label):
return augment(img), label
return augment_with_labels if with_labels else augment
def build_dataset(paths, labels=None, bsize=32, cache=True,
decode_fn=None, augment_fn=None,
augment=True, repeat=True, shuffle=1024,
cache_dir=""):
if cache_dir != "" and cache is True:
os.makedirs(cache_dir, exist_ok=True)
 if decode_fn is None:
decode_fn = build_decoder(labels is not None)
 if augment_fn is None:
augment_fn = build_augmenter(labels is not None)
 AUTO = tf.data.experimental.AUTOTUNE
slices = paths if labels is None else(paths, labels)
 dset = tf.data.Dataset.from_tensor_slices(slices)
 dset = dset.map(decode_fn, num_parallel_calls=AUTO)
 dset = dset.cache(cache_dir)if cache else dset
dset = dset.map(augment_fn, num_parallel_calls=AUTO)if augment else dset
dset = dset.repeat() if repeat else dset
dset = dset.shuffle(shuffle)if shuffle else dset
dset = dset.batch(bsize ).prefetch(AUTO)
 return dset<init_hyperparams>
adf_clf = AdaBoostRegressor(DecisionTreeRegressor(max_depth=3), n_estimators=3000, learning_rate=0.05,
loss='square')
 score = rmse_cv(adf_clf, X_prepared, y_train)
 score.mean()
class TrainConfig(BaseConfig):
EPOCH = 3
FOLDS = 5
TTA = 5
VERBOSITY = 0
WORKERS = 2
LABEL_SMOOTH = 0.0
MULTIPROCESS = False
LR_RATE = {
'0' : 1e-3,
'1' : 1e-3,
'2' : 1e-4,
'3' : 1e-4,
'4' : 1e-4
}
BATCH_SIZE = {
'0' : 32,
'1' : 128,
'2' : 86,
'3' : 128,
'4' : 128
}
IMG_SIZE = {
'0': 850,
'1': 240,
'2': 260,
'3': 224,
'4': 240
}
BASE_NETS = {
'0' : [efn.EfficientNetB0,
'noisy-student'],
'1' : [efn.EfficientNetB1,
'noisy-student'],
'2' : [efn.EfficientNetB2,
'imagenet'],
'3' : [efn.EfficientNetB0,
'imagenet'],
'4' : [efn.EfficientNetB1,
'imagenet'],
}<import_modules>
gbrt = GradientBoostingRegressor(max_depth=3, n_estimators=3000, learning_rate=0.05,
max_features='sqrt', loss='huber')
 gbrt.fit(X_prepared, y_train)
 errors = [mean_squared_error(y_train, y_pred)for y_pred in gbrt.staged_predict(X_prepared)]
bst_n_estimators = np.argmin(errors)
 bst_n_estimators
class SpatialAttentionModule(tf.keras.layers.Layer):
def __init__(self, kernel_size=3):

 super(SpatialAttentionModule, self ).__init__()
self.conv1 = tf.keras.layers.Conv2D(64, kernel_size=kernel_size,
use_bias=False,
kernel_initializer='he_normal',
strides=1, padding='same',
activation=tf.nn.relu6)
 self.conv2 = tf.keras.layers.Conv2D(32, kernel_size=kernel_size,
use_bias=False,
kernel_initializer='he_normal',
strides=1, padding='same',
activation=tf.nn.relu6)
 self.conv3 = tf.keras.layers.Conv2D(16, kernel_size=kernel_size,
use_bias=False,
kernel_initializer='he_normal',
strides=1, padding='same',
activation=tf.nn.relu6)
 self.conv4 = tf.keras.layers.Conv2D(1, kernel_size=kernel_size,
use_bias=False,
kernel_initializer='he_normal',
strides=1, padding='same',
activation=tf.math.sigmoid)
 def call(self, inputs):
avg_out = tf.reduce_mean(inputs, axis=3)
 max_out = tf.reduce_max(inputs, axis=3)
 x = tf.stack([avg_out, max_out], axis=3)
 x = self.conv1(x)
 x = self.conv2(x)
 x = self.conv3(x)
 return self.conv4(x)
 class ChannelAttentionModule(tf.keras.layers.Layer):
def __init__(self, ratio=8):

 super(ChannelAttentionModule, self ).__init__()
self.ratio = ratio
self.gapavg = tf.keras.layers.GlobalAveragePooling2D()
self.gmpmax = tf.keras.layers.GlobalMaxPooling2D()
def build(self, input_shape):
self.conv1 = tf.keras.layers.Conv2D(input_shape[-1]//self.ratio,
kernel_size=1,
strides=1, padding='same',
use_bias=True, activation=tf.nn.relu)
 self.conv2 = tf.keras.layers.Conv2D(input_shape[-1],
kernel_size=1,
strides=1, padding='same',
use_bias=True, activation=tf.nn.relu)
 super(ChannelAttentionModule, self ).build(input_shape)
 def call(self, inputs):
gapavg = self.gapavg(inputs)
 gmpmax = self.gmpmax(inputs)
 gapavg = tf.keras.layers.Reshape(( 1, 1, gapavg.shape[1]))(gapavg)
 gmpmax = tf.keras.layers.Reshape(( 1, 1, gmpmax.shape[1]))(gmpmax)
 gapavg_out = self.conv2(self.conv1(gapavg))
gmpmax_out = self.conv2(self.conv1(gmpmax))
return tf.math.sigmoid(gapavg_out + gmpmax_out)
 def get_output_shape_for(self, input_shape):
return self.compute_output_shape(input_shape)
 def compute_output_shape(self, input_shape):
output_len = input_shape[3]
return(input_shape[0], output_len )<load_pretrained>
gbrt = GradientBoostingRegressor(max_depth=3, n_estimators=bst_n_estimators, learning_rate=0.05, max_features='sqrt',
loss='huber')
 score = rmse_cv(gbrt, X_prepared, y_train)
 score.mean()
class AttentionWeightedAverage2D(tf.keras.layers.Layer):
def __init__(self, **kwargs):
self.init = tf.keras.initializers.get('uniform')
 super(AttentionWeightedAverage2D, self ).__init__(** kwargs)
 def build(self, input_shape):
self.input_spec = [tf.keras.layers.InputSpec(ndim=4)]
assert len(input_shape)== 4
self.W = self.add_weight(shape=(input_shape[3], 1),
name='{}_W'.format(self.name),
initializer=self.init)
 self._trainable_weights = [self.W]
super(AttentionWeightedAverage2D, self ).build(input_shape)
 def call(self, x):
logits = K.dot(x, self.W)
 x_shape = K.shape(x)
 logits = K.reshape(logits,(x_shape[0], x_shape[1], x_shape[2]))
ai = K.exp(logits - K.max(logits, axis=[1,2], keepdims=True))
att_weights = ai /(K.sum(ai, axis=[1,2], keepdims=True)+ K.epsilon())
 weighted_input = x * K.expand_dims(att_weights)
 result = K.sum(weighted_input, axis=[1,2])
 return result
def get_output_shape_for(self, input_shape):
return self.compute_output_shape(input_shape)
 def compute_output_shape(self, input_shape):
output_len = input_shape[3]
return(input_shape[0], output_len )<choose_model_class>
adf_clf.fit(X_prepared, y_train)
 gbrt.fit(X_prepared, y_train)
 lasso_reg.fit(X_prepared, y_train)
 elastic_net.fit(X_prepared, y_train)
 random_reg.fit(X_prepared, y_train)
 ridge_reg.fit(X_prepared, y_train)
 svm_reg.fit(X_prepared, y_train)
 ransac_reg.fit(X_prepared, y_train )
class RANZCRClassifier(tf.keras.Model):
def __init__(self, dim):
super(RANZCRClassifier, self ).__init__()
self.Base = efn.EfficientNetB5(
 input_shape=(TrainConfig.IMG_SIZE['0'],
TrainConfig.IMG_SIZE['0'], 3),
weights=None,
include_top=False)
 self.GAP1 = tf.keras.layers.GlobalAveragePooling2D()
self.GAP2 = tf.keras.layers.GlobalAveragePooling2D()
self.BAT = tf.keras.layers.BatchNormalization()
self.ADD = tf.keras.layers.Add()
self.AVG = tf.keras.layers.Average()
self.DROP = tf.keras.layers.Dropout(rate=0.5)
 self.CAN = ChannelAttentionModule()
self.SPN1 = SpatialAttentionModule()
self.SPN2 = SpatialAttentionModule()
self.AWG = AttentionWeightedAverage2D()
self.DENS = tf.keras.layers.Dense(512, activation=tf.nn.relu)
 self.OUT = tf.keras.layers.Dense(len(target_cols),
activation='sigmoid',
dtype=tf.float32)
 def call(self, input_tensor, training=False):
x = self.Base(input_tensor)
 canx = self.CAN(x)*x
spnx = self.SPN1(canx)*canx
gapx = self.GAP1(spnx)
 wvgx = self.GAP2(self.SPN2(canx))
gapavg = self.AVG([gapx, wvgx])
 awgavg = self.AWG(x)
 x = self.ADD([gapavg, awgavg])
 x = self.BAT(x)
 x = self.DENS(x)
 x = self.DROP(x, training=training)
 return self.OUT(x)
 def build_graph(self):
x = tf.keras.layers.Input(shape=(TrainConfig.IMG_SIZE['0'],
TrainConfig.IMG_SIZE['0'],3))
return tf.keras.Model(inputs=[x], outputs=self.call(x))
model = RANZCRClassifier(( TrainConfig.IMG_SIZE['0'],
TrainConfig.IMG_SIZE['0'], 3))
tf.keras.utils.plot_model(
 model.build_graph() , show_shapes=True,
show_layer_names=True, expand_nested=False
)<load_pretrained>
voting_reg = VotingRegressor([ 
("gradient_boost", gbrt), 
("lasso_reg", lasso_reg), 
("random_reg", random_reg), 
("elastic_net", elastic_net),
])
 score = rmse_cv(voting_reg, X_prepared, y_train)
 score.mean()
model = RANZCRClassifier(( TrainConfig.IMG_SIZE['0'],TrainConfig.IMG_SIZE['0'], 3))
model.build(( None, *(TrainConfig.IMG_SIZE['0'],TrainConfig.IMG_SIZE['0'], 3)))
 model.load_weights('.. /input/multiattentioncheckwg/model.h5' )<categorify>
voting_reg.fit(X_prepared, y_train )
test_paths = BaseConfig.TEST_IMG_PATH + submit['StudyInstanceUID'] + '.jpg'
test_decoder = build_decoder(with_labels=False,
target_size=(TrainConfig.IMG_SIZE['0'],
TrainConfig.IMG_SIZE['0']))
dtest = build_dataset(
 test_paths, bsize=TrainConfig.BATCH_SIZE['0'],
repeat=False, shuffle=False, augment=False,
cache=False, decode_fn=test_decoder
)<save_to_csv>
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y=None):
X = X.values
y = y.values
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=42)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def get_metafeatures(self, X):
return np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )
submit[target_cols] = model.predict(dtest, verbose=1)
 submit.to_csv('submission.csv', index=False)
 submit.head()<set_options>
stacked_averaged_models = StackingAveragedModels(base_models =(elastic_net, gbrt, ridge_reg, random_reg),
meta_model = lasso_reg)
 score = rmse_cv(stacked_averaged_models, X_prepared, y_train)
 score.mean()
init_notebook_mode(connected=True)
 %matplotlib inline 
<set_options>
test_data = setMissingData(test_data)
 test_data = changeSeqCat(test_data)
 test_data = handleSkew(test_data )
env = twosigmanews.make_env()<prepare_x_and_y>
test_prepared = full_pipeline.transform(test_data)
 test_prepared = pd.DataFrame(test_prepared )
( market_train_df, news_train_df)= env.get_training_data()<feature_engineering>
stacked_averaged_models.fit(X_prepared, y_train)
 stack_y_pred = stacked_averaged_models.predict(test_prepared)
 voting_reg_y_pred = voting_reg.predict(test_prepared )
def fix_time_col(in_mkt_df, in_news_df):
if "datetime" in str(in_mkt_df["time"].dtype):
in_mkt_df["time"] = in_mkt_df["time"].dt.date
in_news_df["time"] = in_news_df["time"].dt.date
def clean_assetName_col(in_mkt_df, in_news_df):
in_mkt_df["assetName"] = in_mkt_df["assetName"].str.lower()
in_mkt_df["assetName"] = in_mkt_df["assetName"].str.strip()
in_mkt_df["assetName"] = in_mkt_df["assetName"].str.replace("inc","")
 in_mkt_df["assetName"] = in_mkt_df["assetName"].str.replace("llc","")
 in_mkt_df["assetName"] = in_mkt_df["assetName"].str.replace("ltd","")
 in_mkt_df["assetName"] = in_mkt_df["assetName"].str.strip()
in_news_df["assetName"] = in_news_df["assetName"].str.lower()
in_news_df["assetName"] = in_news_df["assetName"].str.strip()
in_news_df["assetName"] = in_news_df["assetName"].str.replace("inc","")
 in_news_df["assetName"] = in_news_df["assetName"].str.replace("llc","")
 in_news_df["assetName"] = in_news_df["assetName"].str.replace("ltd","")
 in_news_df["assetName"] = in_news_df["assetName"].str.strip()<define_variables>
y_pred = np.expm1(stack_y_pred)* 0.75 + np.expm1(voting_reg_y_pred)* 0.25
def get_final_assetName(row):
min_mkt_assetNameSNo = row["min_mkt_assetNameSNo"]
max_mkt_assetNameSNo = row["max_mkt_assetNameSNo"]
min_assetNameSNo = row["min_assetNameSNo"]
max_assetNameSNo = row["max_assetNameSNo"]
if(( not math.isnan(min_mkt_assetNameSNo)) &(math.isnan(max_mkt_assetNameSNo))):
return min_assetNameSNo
elif(( not math.isnan(max_mkt_assetNameSNo)) &(math.isnan(min_mkt_assetNameSNo))):
return max_assetNameSNo
else:
return 0
def get_replace_assetName(row):
min_mkt_assetNameSNo = row["min_mkt_assetNameSNo"]
max_mkt_assetNameSNo = row["max_mkt_assetNameSNo"]
min_assetNameSNo = row["min_assetNameSNo"]
max_assetNameSNo = row["max_assetNameSNo"]
if(( math.isnan(min_mkt_assetNameSNo)) &(not math.isnan(max_mkt_assetNameSNo))):
return min_assetNameSNo
elif(( math.isnan(max_mkt_assetNameSNo)) &(not math.isnan(min_mkt_assetNameSNo))):
return max_assetNameSNo
else:
return 0
dic_assetName={}
dic_assetNameSNo={}
dic_assetCodes={}
def set_assetNameCodes_SNo(in_mkt_df, in_news_df,dic_assetName, dic_assetNameSNo, dic_assetCodes):
set_asset = set(in_news_df["assetName"].unique())| set(in_mkt_df["assetName"].unique())
 set_codes = set(in_news_df["assetCodes"].unique())
 list_asset = list(set_asset)
 list_codes = list(set_codes)
 if len(dic_assetName)== 0:
dic_assetName= {k:v for v,k in enumerate(list_asset)}
dic_assetNameSNo= {v:k for v,k in enumerate(list_asset)}
dic_assetCodes= {k:v for v,k in enumerate(list_codes)}
else:
set_asset_new = set(dic_assetName.keys())- set_asset
asset_max_SNo = len(dic_assetName)
 dic_assetName_new = {k:v+asset_max_SNo for v,k in enumerate(list(set_asset_new)) }
dic_assetNameSNo_new = {v+asset_max_SNo:k for v,k in enumerate(list(set_asset_new)) }
dic_assetName.update(dic_assetName_new)
 dic_assetNameSNo.update(dic_assetNameSNo_new)
 set_codes_new = set(dic_assetCodes.keys() -set_codes)
 codes_max_SNo = len(dic_assetCodes)
 dic_assetCodes_new = {k:v+codes_max_SNo for v,k in enumerate(list(set_codes_new)) }
dic_assetCodes.update(dic_assetCodes_new)
 in_news_df["assetNameSNo"] = in_news_df["assetName"].map(dic_assetName)
 in_mkt_df["assetNameSNo"] = in_mkt_df["assetName"].map(dic_assetName)
 in_news_df["assetCodesSNo"] = in_news_df["assetCodes"].map(dic_assetCodes)
 return dic_assetName, dic_assetNameSNo, dic_assetCodes<groupby>
submission = pd.DataFrame({
"Id": id_test,
"SalePrice": y_pred
})
 submission
<normalization><EOS> 
submission.to_csv('prediction.csv', index=False )