Kaggle competition: New York City Taxi Trip Duration
<SOS> metric: RMSLE Kaggle data source: new-york-city-taxi-trip-duration<prepare_x_and_y>
import pandas as pd
import numpy as np
from zipfile import ZipFile
import os
from datetime import datetime
import matplotlib.pyplot as plt
import seaborn as sns
from geopy import distance
import math
from sklearn.preprocessing import LabelEncoder, StandardScaler
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation
from keras.optimizers import Adam, SGD
from keras.callbacks import ModelCheckpoint
y = train['FVC'].values
z = train[FE].values
ze = sub[FE].values
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))<choose_model_class>
base_path = ".. /input/nyc-taxi-trip-duration/"
NFOLD = 5
kf = KFold(n_splits=NFOLD )<find_best_model_class>
data_df = pd.read_csv(base_path+"train.csv" )
%%time
cnt = 0
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model()
net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=800,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD<compute_test_metric>
data_df[(data_df['trip_duration']>500000)].count()
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )<feature_engineering>
data_df.drop(data_df[data_df['trip_duration'] > 500000].index, inplace=True )
sub['FVC1'] = 0.996 * pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]<prepare_output>
data_df['trip_duration'] = data_df['trip_duration'].apply(math.log )
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()<feature_engineering>
data_df = data_df[(data_df['trip_duration']<100000)]
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']<load_from_csv>
data_df.reset_index(drop = True, inplace=True )
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1<save_to_csv>
def strtodatetime(x):
return datetime.strptime(x, "%Y-%m-%d %H:%M:%S" )
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )<set_options>
data_df['pickup_datetime'] = data_df['pickup_datetime'].apply(strtodatetime )
HTML('<center><iframe width="560" height="315" src="https://www.youtube.com/embed/AfK9LPNj-Zo" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe></center>' )<set_options>
def dayofweek(x):
return x.weekday()
%matplotlib inline
!pip install chart_studio
cufflinks.go_offline()
cufflinks.set_config_file(world_readable=True, theme='pearl')
 sns.set(style="whitegrid")
 warnings.filterwarnings('ignore')
 plt.style.use('fivethirtyeight')
 plt.show()<load_from_csv>
data_df['day_of_week'] = data_df['pickup_datetime'].apply(dayofweek )
IMAGE_PATH = ".. /input/osic-pulmonary-fibrosis-progressiont/"
train_df = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv')
 test_df = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 print(Fore.YELLOW + 'Training data shape: ',Style.RESET_ALL,train_df.shape)
 train_df.head(5 )<count_missing_values>
distance.distance(( data_df['pickup_latitude'].iloc[0],
data_df['pickup_longitude'].iloc[0]), 
(data_df['dropoff_latitude'].iloc[0],
data_df['dropoff_longitude'].iloc[0])).m
train_df.isnull().sum()<count_missing_values>
def dist(x):
return distance.distance(( x[0], x[1]), 
(x[2], x[3])).m
test_df.isnull().sum()<count_values>
data_df['dist'] = data_df[['pickup_latitude',
'pickup_longitude',
'dropoff_latitude',
'dropoff_longitude']].apply(lambda x:dist(x), axis=1 )
print(Fore.YELLOW +"Total Patients in Train set: ",Style.RESET_ALL,train_df['Patient'].count())
 print(Fore.BLUE +"Total Patients in Test set: ",Style.RESET_ALL,test_df['Patient'].count() )<count_values>
data_df[(data_df['pickup_latitude']==data_df['dropoff_latitude'])& 
(data_df['pickup_longitude']==data_df['dropoff_longitude'])].count()
print(Fore.YELLOW + "The total patient ids are",Style.RESET_ALL,f"{train_df['Patient'].count() },", Fore.BLUE + "from those the unique ids are", Style.RESET_ALL, f"{train_df['Patient'].value_counts().shape[0]}." )<count_values>
train_df['Patient'].value_counts().max()<count_values>
data_df['dist'].replace(to_replace=0, value=1, inplace=True )
test_df['Patient'].value_counts().max()<count_values>
data_df['dist'] = data_df['dist'].apply(math.log )
np.quantile(train_df['Patient'].value_counts() , 0.75)- np.quantile(test_df['Patient'].value_counts() , 0.25 )<count_values>
MAX_SECONDS_IN_DAY = 24*60*60
def timetosectosincosday(x):
initial_date = datetime(x.year, x.month, x.day, 0 , 0, 0)
 sec =(x-initial_date ).total_seconds()
sin = math.sin(2*math.pi*(sec/MAX_SECONDS_IN_DAY))
cos = math.cos(2*math.pi*(sec/MAX_SECONDS_IN_DAY))
return sin, cos
print(np.quantile(train_df['Patient'].value_counts() , 0.95))
print(np.quantile(test_df['Patient'].value_counts() , 0.95))<define_variables>
data_df['pickup_sin_sec'] ,data_df['pickup_cos_sec'] = zip(*data_df['pickup_datetime'].map(timetosectosincosday))
files = folders = 0
path = "/kaggle/input/osic-pulmonary-fibrosis-progression/train"
for _, dirnames, filenames in os.walk(path):
files += len(filenames)
 folders += len(dirnames)
 print(Fore.YELLOW +f'{files:,}',Style.RESET_ALL,"files/images, " + Fore.BLUE + f'{folders:,}',Style.RESET_ALL ,'folders/patients' )<remove_duplicates>
MAX_DAY_IN_WEEK = 6
def dayofweektosincosday(day):
sin = math.sin(2*math.pi*(day/MAX_DAY_IN_WEEK))
cos = math.cos(2*math.pi*(day/MAX_DAY_IN_WEEK))
return sin, cos
patient_df = train_df[['Patient', 'Age', 'Sex', 'SmokingStatus']].drop_duplicates()
patient_df.head()<define_variables>
data_df['sin_day'] ,data_df['cos_day'] = zip(*data_df['day_of_week'].map(dayofweektosincosday))
train_dir = '.. /input/osic-pulmonary-fibrosis-progression/train/'
test_dir = '.. /input/osic-pulmonary-fibrosis-progression/test/'
patient_ids = os.listdir(train_dir)
 patient_ids = sorted(patient_ids)
 no_of_instances = []
age = []
sex = []
smoking_status = []
for patient_id in patient_ids:
patient_info = train_df[train_df['Patient'] == patient_id].reset_index()
no_of_instances.append(len(os.listdir(train_dir + patient_id)))
 age.append(patient_info['Age'][0])
 sex.append(patient_info['Sex'][0])
 smoking_status.append(patient_info['SmokingStatus'][0])
 patient_df = pd.DataFrame(list(zip(patient_ids, no_of_instances, age, sex, smoking_status)) ,
columns =['Patient', 'no_of_instances', 'Age', 'Sex', 'SmokingStatus'])
 print(patient_df.info())
 patient_df.head()<count_values>
df_processed = data_df[['vendor_id',
'passenger_count',
'pickup_sin_sec',
'pickup_cos_sec',
'pickup_longitude',
'pickup_latitude',
'dropoff_longitude',
'dropoff_latitude',
'sin_day',
'cos_day',
'dist',
'trip_duration']]
patient_df['SmokingStatus'].value_counts()<count_values>
X = data_df[['vendor_id',
'passenger_count',
'pickup_sin_sec',
'pickup_cos_sec',
'pickup_longitude',
'pickup_latitude',
'dropoff_longitude',
'dropoff_latitude',
'sin_day',
'cos_day',
'dist']]
train_df['Weeks'].value_counts().head()<count_values>
y = data_df[['trip_duration']]
train_df['FVC'].value_counts()<count_values>
standardScalarX = StandardScaler().fit(X)
 X = standardScalarX.transform(X )
train_df['Percent'].value_counts()<count_values>
standardScalarY = StandardScaler().fit(y)
 y = standardScalarY.transform(y )
patient_df['SmokingStatus'].value_counts()<count_values>
model = Sequential()
model.add(Dense(128, activation = 'relu',input_shape=(X.shape[1],)))
 model.add(Dense(128, activation = 'relu'))
model.add(Dense(64, activation = 'relu'))
model.add(Dense(64, activation = 'relu'))
model.add(Dense(1))
patient_df['Sex'].value_counts()<normalization>
model.compile(optimizer=Adam(lr=0.00001),
metrics=['mean_squared_error'],
loss='mean_squared_error' )
apply_resample = False
def load_scan(path):
slices = [pydicom.read_file(path + '/' + s)for s in os.listdir(path)]
slices.sort(key = lambda x: float(x.ImagePositionPatient[2]))
try:
slice_thickness = np.abs(slices[0].ImagePositionPatient[2] - slices[1].ImagePositionPatient[2])
 except:
slice_thickness = np.abs(slices[0].SliceLocation - slices[1].SliceLocation)
 for s in slices:
s.SliceThickness = slice_thickness
return slices<normalization>
history = model.fit(X, y,
batch_size=32,
validation_split=0.2,
epochs=50,
callbacks=callbacks_list )
def load_scan(path):
slices = [pydicom.read_file(path + '/' + s)for s in os.listdir(path)]
slices.sort(key = lambda x: float(x.ImagePositionPatient[2]))
try:
slice_thickness = np.abs(slices[0].ImagePositionPatient[2] - slices[1].ImagePositionPatient[2])
 except:
slice_thickness = np.abs(slices[0].SliceLocation - slices[1].SliceLocation)
 for s in slices:
s.SliceThickness = slice_thickness
return slices<data_type_conversions>
model.save('model.hdf5' )
def get_pixels_hu(slices):
image = np.stack([s.pixel_array for s in slices])
 image = image.astype(np.int16)
 image[image == -2000] = 0
for slice_number in range(len(slices)) :
intercept = slices[slice_number].RescaleIntercept
slope = slices[slice_number].RescaleSlope
if slope != 1:
image[slice_number] = slope * image[slice_number].astype(np.float64)
 image[slice_number] = image[slice_number].astype(np.int16)
 image[slice_number] += np.int16(intercept)
 return np.array(image, dtype=np.int16 )<data_type_conversions>
test_df = pd.read_csv(base_path+"test.csv" )
def set_lungwin(img, hu=[-1200., 600.]):
lungwin = np.array(hu)
 newimg =(img-lungwin[0])/(lungwin[1]-lungwin[0])
 newimg[newimg < 0] = 0
newimg[newimg > 1] = 1
newimg =(newimg * 255 ).astype('uint8')
 return newimg<load_pretrained>
test_df['pickup_datetime'] = test_df['pickup_datetime'].apply(strtodatetime )
scans = load_scan('.. /input/osic-pulmonary-fibrosis-progression/train/ID00007637202177411956430/')
 scan_array = set_lungwin(get_pixels_hu(scans))<normalization>
test_df['day_of_week'] = test_df['pickup_datetime'].apply(dayofweek )
def resample(imgs, spacing, new_spacing):
new_shape = np.round(imgs.shape * spacing / new_spacing)
 true_spacing = spacing * imgs.shape / new_shape
resize_factor = new_shape / imgs.shape
imgs = zoom(imgs, resize_factor, mode='nearest')
 return imgs, true_spacing, new_shape
spacing_z =(scans[-1].ImagePositionPatient[2] - scans[0].ImagePositionPatient[2])/ len(scans)
 if apply_resample:
scan_array_resample = resample(scan_array, np.array(np.array([spacing_z, *scans[0].PixelSpacing])) , np.array([1.,1.,1.])) [0]<set_options>
test_df['sin_day'] ,test_df['cos_day'] = zip(*test_df['day_of_week'].map(dayofweektosincosday))
imageio.mimsave("/tmp/gif.gif", scan_array, duration=0.0001)
 Image(filename="/tmp/gif.gif", format='png' )<data_type_conversions>
test_df['dist'] = test_df[['pickup_latitude',
'pickup_longitude',
'dropoff_latitude',
'dropoff_longitude']].apply(lambda x:dist(x), axis=1 )
HTML(ani.to_jshtml() )<data_type_conversions>
test_df['dist'].replace(to_replace=0, value=1, inplace=True )
HTML(ani.to_html5_video() )<load_pretrained>
test_df['dist'] = test_df['dist'].apply(math.log )
def extract_dicom_meta_data(filename: str)-> Dict:
image_data = pydicom.read_file(filename)
 img=np.array(image_data.pixel_array ).flatten()
row = {
'Patient': image_data.PatientID,
'body_part_examined': image_data.BodyPartExamined,
'image_position_patient': image_data.ImagePositionPatient,
'image_orientation_patient': image_data.ImageOrientationPatient,
'photometric_interpretation': image_data.PhotometricInterpretation,
'rows': image_data.Rows,
'columns': image_data.Columns,
'pixel_spacing': image_data.PixelSpacing,
'window_center': image_data.WindowCenter,
'window_width': image_data.WindowWidth,
'modality': image_data.Modality,
'StudyInstanceUID': image_data.StudyInstanceUID,
'SeriesInstanceUID': image_data.StudyInstanceUID,
'StudyID': image_data.StudyInstanceUID,
'SamplesPerPixel': image_data.SamplesPerPixel,
'BitsAllocated': image_data.BitsAllocated,
'BitsStored': image_data.BitsStored,
'HighBit': image_data.HighBit,
'PixelRepresentation': image_data.PixelRepresentation,
'RescaleIntercept': image_data.RescaleIntercept,
'RescaleSlope': image_data.RescaleSlope,
'img_min': np.min(img),
'img_max': np.max(img),
'img_mean': np.mean(img),
'img_std': np.std(img)}
return row<prepare_output>
test_df['pickup_sin_sec'] ,test_df['pickup_cos_sec'] = zip(*test_df['pickup_datetime'].map(timetosectosincosday))
meta_data_df = pd.DataFrame.from_dict(meta_data_df)
 meta_data_df.head()<create_dataframe>
test_df_processed = test_df[['vendor_id',
'passenger_count',
'pickup_sin_sec',
'pickup_cos_sec',
'pickup_longitude',
'pickup_latitude',
'dropoff_longitude',
'dropoff_latitude',
'sin_day',
'cos_day',
'dist']]
folder='train'
PATH='.. /input/osic-pulmonary-fibrosis-progression/'
last_index = 2
column_names = ['image_name', 'dcm_ImageOrientationPatient',
'dcm_ImagePositionPatient', 'dcm_PatientID',
'dcm_PatientName', 'dcm_PatientSex'
'dcm_rows', 'dcm_columns']
def extract_DICOM_attributes(folder):
patients_folder = list(os.listdir(os.path.join(PATH, folder)))
 df = pd.DataFrame()
i = 0
for patient_id in patients_folder:
img_path = os.path.join(PATH, folder, patient_id)
 print(img_path)
 images = list(os.listdir(img_path))
for image in images:
image_name = image.split(".")[0]
dicom_file_path = os.path.join(img_path,image)
 dicom_file_dataset = pydicom.read_file(dicom_file_path)

imageOrientationPatient = dicom_file_dataset.ImageOrientationPatient
patientID = dicom_file_dataset.PatientID
patientName = dicom_file_dataset.PatientName
patientSex = dicom_file_dataset.PatientSex
rows = dicom_file_dataset.Rows
cols = dicom_file_dataset.Columns
temp_dict = {'image_name': image_name,
'dcm_ImageOrientationPatient': imageOrientationPatient,
'dcm_PatientID': patientID,
'dcm_PatientName': patientName,
'dcm_PatientSex': patientSex,
'dcm_rows': rows,
'dcm_columns': cols}
df = df.append([temp_dict])
 i += 1
if i == last_index:
break
return df<drop_column>
test_data = standardScalarX.transform(test_df_processed )
extract_DICOM_attributes('train' )<import_modules>
y_pred = model.predict(test_data )
import pandas_profiling as pdp<load_from_csv>
y_pred =standardScalarY.inverse_transform(y_pred)
 y_pred = np.exp(y_pred )
train_df = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv')
 test_df = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv' )<install_modules>
result = test_df[['id']]
!pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index
!pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index<set_options>
result['trip_duration'] = pd.DataFrame(data=y_pred, columns=['trip_duration'] )
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )<set_options>
result.to_csv("submission.csv", index = False )