Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<choose_model_class>
!pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index
!pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index
model = build_model(lr = 1e-3, lr_d = 0, units = 128, dr = 0.5 )<predict_on_test>
import os
import cv2
import pydicom
import pandas as pd
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
import random
from tqdm.notebook import tqdm
from sklearn.model_selection import train_test_split, KFold
from sklearn.metrics import mean_absolute_error
from tensorflow_addons.optimizers import RectifiedAdam
from tensorflow.keras import Model
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
from tensorflow.keras.optimizers import Nadam
import seaborn as sns
import plotly.express as px
import plotly.graph_objects as go
from PIL import Image
pred = model.predict(X_val, batch_size = 1024, verbose = 1 )<data_type_conversions>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed )
predictions =(pred > 0.35 ).astype(int )<compute_test_metric>
seed_everything(42 )
print(metrics.accuracy_score(y_val,predictions))
print(metrics.f1_score(y_val,predictions))<save_to_csv>
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=config )
pred = model.predict(X_test, batch_size = 1024, verbose = 1)
 predictions =(pred > 0.37 ).astype(int)
 sub['prediction'] = predictions
sub.to_csv("submission.csv", index=False )<load_from_csv>
Dropout_model = 0.385
FVC_weight = 0.2
Confidence_weight = 0.2
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
sample_submission = pd.read_csv("/kaggle/input/shelter-animal-outcomes/sample_submission.csv.gz")
 train = pd.read_csv("/kaggle/input/shelter-animal-outcomes/train.csv.gz")
 test = pd.read_csv("/kaggle/input/shelter-animal-outcomes/test.csv.gz")
 print(train.describe())
 print(train.head())
<data_type_conversions>
commits_df = pd.DataFrame(columns = ['commit_num', 'FVC_weight', 'Dropout_model', 'LB_score', 'seed'] )
def binarize_name(df):
df = df.rename(columns={'Name': 'HasName'})
 df['HasName'] = pd.to_numeric(df['HasName'].str.isalpha())
 df['HasName'] = df['HasName'].fillna(0)
 return df
train = binarize_name(train)
 test = binarize_name(test)
 train['SexuponOutcome'] = train['SexuponOutcome'].fillna('Unknown')
 test['SexuponOutcome'] = test['SexuponOutcome'].fillna('Unknown')
 train = train.fillna({'AgeuponOutcome': '9999 days'})
 test = test.fillna({'AgeuponOutcome': '9999 days'})
 def string_to_num_days(series):
split_series = series.str.split(" ")
 split_df = pd.DataFrame(split_series.to_list() , columns=['multiplier', 'days'])
 convert_days = {'year': 365, 'years': 365, 'months': 30.5, 'month': 30.5, 'week': 7, 'weeks': 7, 'days': 1, 'day':1}
split_df['days'] = split_df['days'].map(convert_days)
 split_df['multiplier'] = split_df['multiplier'].astype(int)
 split_df['total_days'] = split_df['days'] * split_df['multiplier']
return split_df['total_days']
train['AgeuponOutcome'] = string_to_num_days(train['AgeuponOutcome'])
 test['AgeuponOutcome'] = string_to_num_days(test['AgeuponOutcome'])
 train = train.replace({'AgeuponOutcome': {9999: np.nan}})
 train['AgeuponOutcome'] = train['AgeuponOutcome'].fillna(train['AgeuponOutcome'].mean())
 test = test.replace({'AgeuponOutcome': {9999: np.nan}})
 test['AgeuponOutcome'] = test['AgeuponOutcome'].fillna(test['AgeuponOutcome'].mean())
<count_unique_values>
n=0
commits_df.loc[n,'commit_num'] = 1
commits_df.loc[n,'Dropout_model'] = 0.4
commits_df.loc[n,'FVC_weight'] = 0.25
commits_df.loc[n,'LB_score'] = -6.8111
print('Distinct Colors: ', len(train['Color'].unique()))
print('Distinct Breeds: ', len(train['Breed'].unique()))<feature_engineering>
n=1
commits_df.loc[n,'commit_num'] = 4
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.25
commits_df.loc[n,'LB_score'] = -6.8105
def extract_color(df):
df['IsSolidColor'] = 1
mixed_entries = df['Color'].str.contains(r'\/')
 df.loc[mixed_entries, 'IsSolidColor'] = 0
df['Color'] = df['Color'].str.replace('\/.*', '')
 return df
train = extract_color(train)
 test = extract_color(test)
 def extract_markings(df):
for marking in ['Tabby','Point','Tiger','Brindle','Smoke','Tick','Merle']:
df[marking] = df['Color'].str.contains(marking ).astype(int)
 return df
train = extract_markings(train)
 test = extract_markings(test)
 print(train['IsSolidColor'].value_counts())
 for marking in ['Tabby','Point','Tiger','Brindle','Smoke','Tick','Merle']:
print(train[marking].value_counts() )<drop_column>
n=2
commits_df.loc[n,'commit_num'] = 6
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.35
commits_df.loc[n,'LB_score'] = -6.8158
train = train.drop(columns=['Tabby','Point','Tiger','Brindle','Smoke','Tick','Merle'] )<feature_engineering>
n=3
commits_df.loc[n,'commit_num'] = 8
commits_df.loc[n,'Dropout_model'] = 0.35
commits_df.loc[n,'FVC_weight'] = 0.25
commits_df.loc[n,'LB_score'] = -6.8107
def extract_purebred(df):
df['Purebred'] = 1
mixed_entries =(df['Breed'].str.contains('\/')) |(df['Breed'].str.contains('Mix'))
df.loc[mixed_entries, 'Purebred'] = 0
return df
train = extract_purebred(train)
 test = extract_purebred(test)
 def extract_aggression(df):
aggro_breeds = 'Staffordshire|Pit|Doberman|Chow|Rottweiler|German Shepherd|American Bulldog|Mastiff|Bullmastiff|Husky|Malamute|Akita|Boxer'
df['AggroBreed'] = 0
df.loc[df['Breed'].str.contains(aggro_breeds), 'AggroBreed'] = 1
return df
train = extract_aggression(train)
 test = extract_aggression(test)
 def extract_hypoallergenic(df):
hypoallergenic_breeds = 'Affenpinscher|Afghan|Hairless|Barbet|Bedlington|Bichon|Bolognese|Crested|Schnauzer|Water Spaniel|Kerry|Maltese|Poodle|Portuguese Water|Yorkshire'
df['HypoallergenicBreed'] = 0
df.loc[df['Breed'].str.contains(hypoallergenic_breeds), 'HypoallergenicBreed'] = 1
return df
train = extract_hypoallergenic(train)
 test = extract_hypoallergenic(test )<feature_engineering>
n=4
commits_df.loc[n,'commit_num'] = 9
commits_df.loc[n,'Dropout_model'] = 0.35
commits_df.loc[n,'FVC_weight'] = 0.3
commits_df.loc[n,'LB_score'] = -6.8125
train['DateTime'] = pd.to_datetime(train['DateTime'], infer_datetime_format=True)
 test['DateTime'] = pd.to_datetime(test['DateTime'], infer_datetime_format=True)
 train['Hour'] = train['DateTime'].dt.round("H" ).dt.hour
test['Hour'] = test['DateTime'].dt.round("H" ).dt.hour
train['HourSin'] = np.sin(2 * np.pi * train['Hour']/23.0)
 train['HourCos'] = np.cos(2 * np.pi * train['Hour']/23.0)
 test['HourSin'] = np.sin(2 * np.pi * test['Hour']/23.0)
 test['HourCos'] = np.cos(2 * np.pi * test['Hour']/23.0)
 train['DayOfWeek'] = pd.to_datetime(train['DateTime'], infer_datetime_format=True ).dt.weekday.map({0:'Monday', 1:'Tuesday', 2:'Wednesday', 3:'Thursday', 4:'Friday',5:'Saturday', 6:'Sunday'})
 test['DayOfWeek'] = pd.to_datetime(test['DateTime'], infer_datetime_format=True ).dt.weekday.map({0:'Monday', 1:'Tuesday', 2:'Wednesday', 3:'Thursday', 4:'Friday',5:'Saturday', 6:'Sunday'})
<data_type_conversions>
n=5
commits_df.loc[n,'commit_num'] = 10
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8089
train['Dog'] = train['AnimalType'].str.contains('Dog' ).astype(int)
 test['Dog'] = test['AnimalType'].str.contains('Dog' ).astype(int)
 def extract_sex_info(df):
df['SexuponOutcome'].dropna(inplace=True)
 df['Male'] = df['SexuponOutcome'].str.contains("Male" ).astype(int)
 df['Intact'] = df['SexuponOutcome'].str.contains("Intact" ).astype(int)
 df['SexUnknown'] = df['SexuponOutcome'].str.contains("Unknown" ).astype(int)
 return df
train = extract_sex_info(train)
 test = extract_sex_info(test)
<data_type_conversions>
n=6
commits_df.loc[n,'commit_num'] = 11
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.15
commits_df.loc[n,'LB_score'] = -6.8100
print(train[train['Hour'] == 0]['DateTime'].dt.strftime("%Y-%m-%d %H:%M:%S"))
print(train[train['Hour'] == 9]['DateTime'].dt.strftime("%Y-%m-%d %H:%M:%S"))<count_values>
n=7
commits_df.loc[n,'commit_num'] = 13
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.175
commits_df.loc[n,'LB_score'] = -6.8096
print('Outcomes for 0:00:00:
', train[(train['OnTheHour'] == 1)&(train['Hour'] == 0)]['OutcomeType'].value_counts() , '
')
 print('Outcomes for 9:00:00:
', train[(train['OnTheHour'] == 1)&(train['Hour'] == 9)]['OutcomeType'].value_counts() )<data_type_conversions>
n=8
commits_df.loc[n,'commit_num'] = 14
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.225
commits_df.loc[n,'LB_score'] = -6.8100
def is_weekend(df):
df['weekend'] = df['DateTime'].dt.weekday.isin([5, 6] ).astype(int)
 df = df.drop(columns=['DayOfWeek'])
 return df
train = is_weekend(train)
 test = is_weekend(test )<prepare_x_and_y>
n=9
commits_df.loc[n,'commit_num'] = 15
commits_df.loc[n,'Dropout_model'] = 0.32
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8092
y_train = train['OutcomeType']
X_train = train[['HasName','AgeuponOutcome', 'AggroBreed','HourSin', 'HourCos', 'OnTheHour', 'Dog', 'Male', 'Intact', 'SexUnknown', 'weekend']]
X_test = test[['HasName','AgeuponOutcome', 'AggroBreed','HourSin', 'HourCos', 'OnTheHour', 'Dog', 'Male', 'Intact', 'SexUnknown', 'weekend']] 
<train_model>
n=10
commits_df.loc[n,'commit_num'] = 16
commits_df.loc[n,'Dropout_model'] = 0.25
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8093
xgb_model = XGBClassifier(random_state = 42, n_estimators = 45)
 xgb_model.fit(X_train, y_train )<save_to_csv>
n=11
commits_df.loc[n,'commit_num'] = 17
commits_df.loc[n,'Dropout_model'] = 0.25
commits_df.loc[n,'FVC_weight'] = 0.5
commits_df.loc[n,'LB_score'] = -6.8283
xgb_pred = xgb_model.predict_proba(X_test)
 results = pd.DataFrame(xgb_pred, columns = xgb_model.classes_, index = test['ID'] ).reset_index()
results.to_csv('submission.csv', index=False )<set_options>
n=12
commits_df.loc[n,'commit_num'] = 18
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8087
%matplotlib inline<load_from_csv>
n=13
commits_df.loc[n,'commit_num'] = 19
commits_df.loc[n,'Dropout_model'] = 0.39
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8090
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 test_ID = test_df['ID']
train_OutcomeType = train_df['OutcomeType']
train_df = train_df.drop(['AnimalID','OutcomeSubtype','OutcomeType'] , axis=1)
 test_df = test_df.drop('ID', axis=1)
 combine_df = pd.concat([train_df, test_df], axis=0 )<data_type_conversions>
n=14
commits_df.loc[n,'commit_num'] = 20
commits_df.loc[n,'Dropout_model'] = 0.37
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8092
combine_df['HasName']=combine_df['Name'].notnull().astype(int )<feature_engineering>
n=15
commits_df.loc[n,'commit_num'] = 21
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.19
commits_df.loc[n,'LB_score'] = -6.8093
interval = pd.to_datetime(combine_df['DateTime'])-pd.to_datetime('2013-01-01')
 combine_df['DateTime_day'] = interval /np.timedelta64(1, 'D')
 interval =(( interval/np.timedelta64(1, 'D')) %1*24)
 interval[(interval>=0)&(interval)<6] = 0
interval[(interval>=6)&(interval<12)] = 1
interval[(interval>=12)&(interval<18)] = 2
interval[(interval>=18)&(interval<22)] = 3
interval[(interval>=22)&(interval<24)] =0
combine_df['DateTime_mmae'] = interval<categorify>
n=16
commits_df.loc[n,'commit_num'] = 23
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8402
combine_df['AnimalType'] = combine_df['AnimalType'].map({'Dog':0, 'Cat':1} )<categorify>
commits_df['LB_score'] = pd.to_numeric(commits_df['LB_score'])
 commits_df['max'] = 0
commits_df.loc[commits_df['LB_score'].idxmax() , 'max'] = 1
combine_df['SexuponOutcome'] = combine_df['SexuponOutcome'].fillna(combine_df['SexuponOutcome'].mode() [0])
 combine_df['Sex'] = combine_df['SexuponOutcome'].map({'Intact Female':0,'Spayed Female':0,'Intact Male':1,'Neutered Male':1,'Unknown':2})
 combine_df['IsIntact'] = combine_df['SexuponOutcome'].map({'Intact Female':0,'Intact Male':0, 'Neutered Male':1,'Spayed Female':1,'Unknown':2})
 combine_df[['SexuponOutcome','Sex','IsIntact']].head()<feature_engineering>
commits_df['seed'] = 42
commits_df.loc[commits_df['commit_num'] == 23, 'seed'] = 0
combine_df['AgeuponOutcome'] = combine_df['AgeuponOutcome'].fillna(combine_df['AgeuponOutcome'].mode() [0])
 def convert(x):
a = str(x ).split(' ')[0]
b = str(x ).split(' ')[-1]
if 'year' in str(b):
return int(a)*12*52*7
elif 'month' in str(b):
return int(a)*4*7
elif 'week' in str(b):
return int(a)*7
else:
return int(a)
 combine_df['AgeuponOutcome'] = combine_df['AgeuponOutcome'].map(convert )<feature_engineering>
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv' )
combine_df['IsMix'] = combine_df['Breed'].map(lambda x: 'Mix' in x ).astype(int)
 combine_df['Breed'] = combine_df['Breed'].map(lambda x:x.strip(' Mix' ).split('/')[0].split(' ')[-1] )<feature_engineering>
def get_tab(df):
vector = [(df.Age.values[0] - 30)/ 30]
if df.Sex.values[0] == 'male':
vector.append(0)
 else:
vector.append(1)
 if df.SmokingStatus.values[0] == 'Never smoked':
vector.extend([0,0])
 elif df.SmokingStatus.values[0] == 'Ex-smoker':
vector.extend([1,1])
 elif df.SmokingStatus.values[0] == 'Currently smokes':
vector.extend([0,1])
 else:
vector.extend([1,0])
 return np.array(vector )
combine_df['IsPure'] = combine_df['Color'].map(lambda x: '/' not in x ).astype(int)
 combine_df[['Color','IsPure']].head()
combine_df['Color'] = combine_df['Color'].map(lambda x: x.split('/')[0].split(' ')[0] )<categorify>
A = {}
TAB = {}
P = []
for i, p in tqdm(enumerate(train.Patient.unique())) :
sub = train.loc[train.Patient == p, :]
fvc = sub.FVC.values
weeks = sub.Weeks.values
c = np.vstack([weeks, np.ones(len(weeks)) ] ).T
a, b = np.linalg.lstsq(c, fvc)[0]
A[p] = a
TAB[p] = get_tab(sub)
 P.append(p )
combine_df_DateTime_mmae = pd.get_dummies(combine_df['DateTime_mmae'], prefix='DateTime_mmae')
 combine_df_breed = pd.get_dummies(combine_df['Breed'], prefix='Breed')
 combine_df_color = pd.get_dummies(combine_df['Color'], prefix='Color')
 combine_df['DateTime_day'] =(combine_df['DateTime_day'] - combine_df['DateTime_day'].min())/(combine_df['DateTime_day'].max() - combine_df['DateTime_day'].min())
 combine_df['AgeuponOutcome'] =(combine_df['AgeuponOutcome'] - combine_df['AgeuponOutcome'].min())/(combine_df['AgeuponOutcome'].max() - combine_df['AgeuponOutcome'].min() )<prepare_x_and_y>
class IGenerator(Sequence):
BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']
def __init__(self, keys, a, tab, batch_size=32):
self.keys = [k for k in keys if k not in self.BAD_ID]
self.a = a
self.tab = tab
self.batch_size = batch_size
self.train_data = {}
for p in train.Patient.values:
self.train_data[p] = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 def __len__(self):
return 1000
def __getitem__(self, idx):
x = []
a, tab = [], []
keys = np.random.choice(self.keys, size = self.batch_size)
 for k in keys:
try:
i = np.random.choice(self.train_data[k], size=1)[0]
img = get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')
 x.append(img)
 a.append(self.a[k])
 tab.append(self.tab[k])
 except:
print(k, i)
 x,a,tab = np.array(x), np.array(a), np.array(tab)
 x = np.expand_dims(x, axis=-1)
 return [x, tab] , a
X = combine_df[['HasName','DateTime_day', 'AnimalType', 'AgeuponOutcome','Sex','IsIntact','IsMix','IsPure']]
X = pd.concat([X, combine_df_breed, combine_df_color, combine_df_DateTime_mmae], axis=1)
 train_x = X.iloc[0:train_df.shape[0],]
test_x = X.iloc[train_df.shape[0]:,]
train_y = train_OutcomeType.map({'Adoption':0,'Died':1,'Euthanasia':2,'Return_to_owner':3,'Transfer':4} )<split>
Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D,
LeakyReLU, Concatenate 
)
 def get_efficientnet(model, shape):
models_dict = {
'b0': efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False),
'b1': efn.EfficientNetB1(input_shape=shape,weights=None,include_top=False),
'b2': efn.EfficientNetB2(input_shape=shape,weights=None,include_top=False),
'b3': efn.EfficientNetB3(input_shape=shape,weights=None,include_top=False),
'b4': efn.EfficientNetB4(input_shape=shape,weights=None,include_top=False),
'b5': efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False),
'b6': efn.EfficientNetB6(input_shape=shape,weights=None,include_top=False),
'b7': efn.EfficientNetB7(input_shape=shape,weights=None,include_top=False)
 }
return models_dict[model]
def build_model(shape=(512, 512, 1), model_class=None):
inp = Input(shape=shape)
 base = get_efficientnet(model_class, shape)
 x = base(inp)
 x = GlobalAveragePooling2D()(x)
 inp2 = Input(shape=(4,))
x2 = tf.keras.layers.GaussianNoise(0.2 )(inp2)
 x = Concatenate()([x, x2])
 x = Dropout(Dropout_model )(x)
 x = Dense(1 )(x)
 model = Model([inp, inp2] , x)
 weights = [w for w in os.listdir('.. /input/osic-model-weights')if model_class in w][0]
model.load_weights('.. /input/osic-model-weights/' + weights)
 return model
model_classes = ['b5']
models = [build_model(shape=(512, 512, 1), model_class=m)for m in model_classes]
print('Number of models: ' + str(len(models)) )
tr_x, va_x, tr_y, va_y = train_test_split(train_x, train_y,test_size=0.3,random_state=0 )<import_modules>
tr_p, vl_p = train_test_split(P, shuffle=True, train_size = 0.8 )
from sklearn.metrics import accuracy_score, log_loss
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC, LinearSVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier
from sklearn.naive_bayes import GaussianNB, MultinomialNB, BernoulliNB
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis<predict_on_test>
def score(fvc_true, fvc_pred, sigma):
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric )
clf1 = KNeighborsClassifier(3)
 clf1.fit(tr_x, tr_y)
 name = clf1.__class__.__name__
print(name),
va_pred = clf1.predict(va_x)
 acc = accuracy_score(va_y, va_pred)
 print("Accuracy: {:.4%}".format(acc)) ,
va_pred = clf1.predict_proba(va_x)
 ll = log_loss(va_y, va_pred)
 print("Log Loss: {:.4f}".format(ll))<compute_train_metric>
subs = []
for model in models:
metric = []
for q in tqdm(range(1, 10)) :
m = []
for p in vl_p:
x = []
tab = []
if p in ['ID00011637202177653955184', 'ID00052637202186188008618']:
continue
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/{i}'))
tab.append(get_tab(train.loc[train.Patient == p, :]))
if len(x)< 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q / 10)
 percent_true = train.Percent.values[train.Patient == p]
fvc_true = train.FVC.values[train.Patient == p]
weeks_true = train.Weeks.values[train.Patient == p]
fvc = a *(weeks_true - weeks_true[0])+ fvc_true[0]
percent = percent_true[0] - a * abs(weeks_true - weeks_true[0])
 m.append(score(fvc_true, fvc, percent))
print(np.mean(m))
metric.append(np.mean(m))
q =(np.argmin(metric)+ 1)/ 10
sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 A_test, B_test, P_test,W, FVC= {}, {}, {},{},{}
STD, WEEK = {}, {}
for p in test.Patient.unique() :
x = []
tab = []
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/{i}'))
tab.append(get_tab(test.loc[test.Patient == p, :]))
if len(x)<= 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q)
 A_test[p] = a
B_test[p] = test.FVC.values[test.Patient == p] - a*test.Weeks.values[test.Patient == p]
P_test[p] = test.Percent.values[test.Patient == p]
WEEK[p] = test.Weeks.values[test.Patient == p]
for k in sub.Patient_Week.values:
p, w = k.split('_')
 w = int(w)
 fvc = A_test[p] * w + B_test[p]
sub.loc[sub.Patient_Week == k, 'FVC'] = fvc
sub.loc[sub.Patient_Week == k, 'Confidence'] =(
 P_test[p] - A_test[p] * abs(WEEK[p] - w)
)
 _sub = sub[["Patient_Week","FVC","Confidence"]].copy()
subs.append(_sub )
clf2 = SVC(kernel="rbf", C=0.025, probability=True)
 clf2.fit(tr_x, tr_y)
 name = clf2.__class__.__name__
print(name),
va_pred = clf2.predict(va_x)
 acc = accuracy_score(va_y, va_pred)
 print("Accuracy: {:.4%}".format(acc)) ,
va_pred = clf2.predict_proba(va_x)
 ll = log_loss(va_y, va_pred)
 print("Log Loss: {:.4f}".format(ll))<compute_train_metric>
N = len(subs)
 sub = subs[0].copy()
sub["FVC"] = 0
sub["Confidence"] = 0
for i in range(N):
sub["FVC"] += subs[0]["FVC"] *(1/N)
 sub["Confidence"] += subs[0]["Confidence"] *(1/N )
clf3 = DecisionTreeClassifier()
clf3.fit(tr_x, tr_y)
 name = clf3.__class__.__name__
print(name),
va_pred = clf3.predict(va_x)
 acc = accuracy_score(va_y, va_pred)
 print("Accuracy: {:.4%}".format(acc)) ,
va_pred = clf3.predict_proba(va_x)
 ll = log_loss(va_y, va_pred)
 print("Log Loss: {:.4f}".format(ll))<compute_train_metric>
sub[["Patient_Week","FVC","Confidence"]].to_csv("submission_img.csv", index=False )
clf4 = RandomForestClassifier()
clf4.fit(tr_x, tr_y)
 name = clf4.__class__.__name__
print(name),
va_pred = clf4.predict(va_x)
 acc = accuracy_score(va_y, va_pred)
 print("Accuracy: {:.4%}".format(acc)) ,
va_pred = clf4.predict_proba(va_x)
 ll = log_loss(va_y, va_pred)
 print("Log Loss: {:.4f}".format(ll))<compute_train_metric>
img_sub = sub[["Patient_Week","FVC","Confidence"]].copy()
clf5 = AdaBoostClassifier()
clf5.fit(tr_x, tr_y)
 name = clf5.__class__.__name__
print(name),
va_pred = clf5.predict(va_x)
 acc = accuracy_score(va_y, va_pred)
 print("Accuracy: {:.4%}".format(acc)) ,
va_pred = clf5.predict_proba(va_x)
 ll = log_loss(va_y, va_pred)
 print("Log Loss: {:.4f}".format(ll))<compute_train_metric>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )
clf6 = GradientBoostingClassifier()
clf6.fit(tr_x, tr_y)
 name = clf6.__class__.__name__
print(name),
va_pred = clf6.predict(va_x)
 acc = accuracy_score(va_y, va_pred)
 print("Accuracy: {:.4%}".format(acc)) ,
va_pred = clf6.predict_proba(va_x)
 ll = log_loss(va_y, va_pred)
 print("Log Loss: {:.4f}".format(ll))<compute_train_metric>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )
clf7 = GaussianNB()
clf7.fit(tr_x, tr_y)
 name = clf7.__class__.__name__
print(name),
va_pred = clf7.predict(va_x)
 acc = accuracy_score(va_y, va_pred)
 print("Accuracy: {:.4%}".format(acc)) ,
va_pred = clf7.predict_proba(va_x)
 ll = log_loss(va_y, va_pred)
 print("Log Loss: {:.4f}".format(ll))<compute_train_metric>
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique() )
clf8 = MultinomialNB()
clf8.fit(tr_x, tr_y)
 name = clf8.__class__.__name__
print(name),
va_pred = clf8.predict(va_x)
 acc = accuracy_score(va_y, va_pred)
 print("Accuracy: {:.4%}".format(acc)) ,
va_pred = clf8.predict_proba(va_x)
 ll = log_loss(va_y, va_pred)
 print("Log Loss: {:.4f}".format(ll))<compute_train_metric>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
clf9 = BernoulliNB()
clf9.fit(tr_x, tr_y)
 name = clf9.__class__.__name__
print(name),
va_pred = clf9.predict(va_x)
 acc = accuracy_score(va_y, va_pred)
 print("Accuracy: {:.4%}".format(acc)) ,
va_pred = clf9.predict_proba(va_x)
 ll = log_loss(va_y, va_pred)
 print("Log Loss: {:.4f}".format(ll))<compute_train_metric>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
clf10 = LinearDiscriminantAnalysis()
clf10.fit(tr_x, tr_y)
 name = clf10.__class__.__name__
print(name),
va_pred = clf10.predict(va_x)
 acc = accuracy_score(va_y, va_pred)
 print("Accuracy: {:.4%}".format(acc)) ,
va_pred = clf10.predict_proba(va_x)
 ll = log_loss(va_y, va_pred)
 print("Log Loss: {:.4f}".format(ll))<choose_model_class>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
model = GradientBoostingClassifier()
model.fit(train_x, train_y)
 test_prediction = model.predict_proba(test_x )<save_to_csv>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )
LABELS = sorted(train_OutcomeType.unique())
 PredictResult = pd.DataFrame(test_prediction,columns=LABELS)
 PredictResult.columns.names = ['ID']
PredictResult.index.names = ['ID']
PredictResult.index += 1
PredictResult.to_csv('GBC_pred_1.csv', index_label='ID' )<train_model>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
class TextTransformer(BaseEstimator, TransformerMixin):
def __init__(self, est):
super(TextTransformer, self ).__init__()
self.est = est
pass
def fit(self, X, y=None):
self.est.fit(X.ravel())
 return self
def transform(self, X):
Xs = [
self.est.transform(X[:,_])
 for _ in range(X.shape[-1])
 ]
result =(Xs[0]>0 ).astype(int)
 for _ in range(len(Xs)-1):
result +=(Xs[_+1]>0 ).astype(int)
 return sp.hstack(( 
(result == len(Xs)).astype(float), 
(result == 1 ).astype(float)
)).tocsr()
class SupervisedTransformer(BaseEstimator, TransformerMixin):
def __init__(self, est, method):
super(SupervisedTransformer, self ).__init__()
self.est = est
self.method = method
pass
def fit(self, X, y=None):
self.est.fit(X, y)
 return self
def transform(self, X):
return getattr(self.est, self.method )(X)
 class EqNotEqBinarizer(BaseEstimator, TransformerMixin):
def __init__(self):
super(EqNotEqBinarizer, self ).__init__()
pass
def fit(self, X, y=None):
assert X.shape[-1] == 2, 'Only two-column arrays'
self.bin_ = LabelBinarizer(sparse_output=True)
 self.bin_.fit(X.ravel())
 return self
def transform(self, X):
z = np.zeros(( X.shape[0], 3), dtype=int)
 eqmask = X[:,0] == X[:,1]
noteqmask = X[:,0] != X[:,1]
z[eqmask,0] = X[eqmask,0]
z[noteqmask,1] = X[noteqmask,0]
z[noteqmask,2] = X[noteqmask,1]
return sp.hstack((
self.bin_.transform(z[:,0]),
self.bin_.transform(z[:,1])+ self.bin_.transform(z[:,2])
))
def get_file(mode):
params = {}
if mode == 'test':
params['index_col'] = 0
items = pd.read_csv('.. /input/ItemInfo_%s.csv' % mode, index_col=0)[[
'categoryID', 'title', 'locationID', 'metroID', 'lon', 'lat']]
items['title'] = items['title'].fillna('nan')
 items['metroID'] = items['metroID'].fillna(-1)
 parent_categories = pd.read_csv('.. /input/Category.csv', index_col=0)
 regions = pd.read_csv('.. /input/Location.csv', index_col=0)
 items = pd.merge(items, parent_categories, left_on='categoryID', right_index=True, how='inner', sort=False)
 items = pd.merge(items, regions, left_on='locationID', right_index=True, how='inner', sort=False)
 del parent_categories
del regions
pr = pd.read_csv('.. /input/ItemPairs_%s.csv' % mode, **params)
 pr = pd.merge(pr, items, left_on='itemID_1', right_index=True, how='inner', sort=False)
 pr = pd.merge(pr, items, left_on='itemID_2', right_index=True, how='inner', sort=False)
 del items
print('Columns: ' + str(pr.columns), file=sys.stderr)
 fields = [
'categoryID_x', 'parentCategoryID_x', 'title_x', 'title_y',
'locationID_x', 'locationID_y', 'regionID_x', 'regionID_y', 'metroID_x', 'metroID_y',
'lon_x', 'lon_y', 'lat_x', 'lat_y',
]
if mode == 'train':
return pr[fields + ['isDuplicate']]
else:
return pr[fields]
def get_balanced_train_indices(column="categoryID"):
prtest = pd.read_csv(".. /input/ItemPairs_test.csv", index_col=0)
 prtest = pd.merge(prtest,
pd.read_csv(".. /input/ItemInfo_test.csv", index_col=0),
left_on="itemID_1", right_index=True, how="inner", sort=False)
 catdist = prtest[column].value_counts() / len(prtest)
 del prtest
prtrain = pd.read_csv(".. /input/ItemPairs_train.csv")
 prtrain = pd.merge(prtrain,
pd.read_csv(".. /input/ItemInfo_train.csv", index_col=0),
left_on="itemID_1", right_index=True, how="inner", sort=False)
 indices = np.array([])
 trainsize = len(prtrain)
 for cat, dist in catdist.iteritems() :
trcatdist = len(prtrain[prtrain[column] == cat])
 if trcatdist < int(1.0 * dist * trainsize):
trainsize = int(1.0 * trainsize * trcatdist /(dist * trainsize))
for cat, dist in catdist.iteritems() :
indices = np.hstack(( indices, shuffle(prtrain[prtrain[column] == cat].index, random_state=1)[:int(dist*trainsize)]))
indices = pd.Index(np.sort(indices.astype(int)))
 return indices
def _print_shape(X):
print("SHAPE: ", X.shape, file=sys.stderr)
 return X
est = Pipeline([ 
('shape1', FunctionTransformer(_print_shape, validate=False)) , 
('feats', FeatureUnion(transformer_list=[ 
('categories', Pipeline([ 
('filter', FunctionTransformer(lambda X: X[:,[0]], validate=False)) , 
('binarizer', OneHotEncoder()), 
('shape1', FunctionTransformer(_print_shape, validate=False)) ,
])) , 
('parentCategories', Pipeline([ 
('filter', FunctionTransformer(lambda X: X[:,[1]], validate=False)) , 
('binarizer', OneHotEncoder()), 
('shape1', FunctionTransformer(_print_shape, validate=False)) ,
])) , 
('titles', Pipeline([ 
('filter', FunctionTransformer(lambda X: X[:,[2,3]], validate=False)) , 
('titleswitch', TextTransformer(CountVectorizer(binary=True))), 
('logreg', SupervisedTransformer(LogisticRegression(C=0.01), 'predict_proba')) , 
('selector', FunctionTransformer(lambda X: X[:,[1]])) , 
('shape1', FunctionTransformer(_print_shape, validate=False)) ,
])) , 
('regionID', Pipeline([ 
('filter', FunctionTransformer(lambda X: X[:,[6,7]].astype(int), validate=False)) , 
('binarizer', EqNotEqBinarizer()), 
('threshold', VarianceThreshold(0.0001)) , 
('shape1', FunctionTransformer(_print_shape, validate=False)) ,
])) , 
('coords', Pipeline([ 
('filter', FunctionTransformer(lambda X: X[:,[10,11,12,13]].astype(float), validate=False)) , 
('shape1', FunctionTransformer(_print_shape, validate=False)) ,
])) ,
])) , 
('shape2', FunctionTransformer(_print_shape, validate=False)) , 
('est', XGBClassifier()),
])
 pr = get_file('train')
 print('Columns: ' + str(pr.columns), file=sys.stderr)
 print('FITTING...', file=sys.stderr)
 est.fit(pr.drop('isDuplicate', axis=1 ).values, pr['isDuplicate'].values)
 print('FITTED', file=sys.stderr)
 del pr
pr = get_file('test')
 print('Columns: ' + str(pr.columns), file=sys.stderr)
 pr['probability'] = est.predict_proba(pr.values)[:,1]
pr[['probability']].to_csv('submission.csv')
<load_from_csv>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
def read_data() :
print('Reading files...')
 calendar = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/calendar.csv')
 calendar = reduce_mem_usage(calendar)
 print('Calendar has {} rows and {} columns'.format(calendar.shape[0], calendar.shape[1]))
sell_prices = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sell_prices.csv')
 sell_prices = reduce_mem_usage(sell_prices)
 print('Sell prices has {} rows and {} columns'.format(sell_prices.shape[0], sell_prices.shape[1]))
sales_train_val = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sales_train_validation.csv')
 print('Sales train validation has {} rows and {} columns'.format(sales_train_val.shape[0], sales_train_val.shape[1]))
submission = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sample_submission.csv')
 return calendar, sell_prices, sales_train_val, submission<prepare_output>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.65), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
calendar, sell_prices, sales_train_val, submission = read_data()<define_variables>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
nh = z.shape[1]
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
NUM_ITEMS = sales_train_val.shape[0]
DAYS_PRED = submission.shape[1] - 1<categorify>
NFOLD = 5
kf = KFold(n_splits=NFOLD )
def encode_categorical(df, cols):
for col in cols:
le = LabelEncoder()
df[col] = df[col].fillna('nan')
 df[col] = pd.Series(le.fit_transform(df[col]), index=df.index)
 return df
calendar = encode_categorical(
 calendar, ["event_name_1", "event_type_1", "event_name_2", "event_type_2"]
).pipe(reduce_mem_usage)
 sales_train_val = encode_categorical(
 sales_train_val, ["item_id", "dept_id", "cat_id", "store_id", "state_id"],
).pipe(reduce_mem_usage)
 sell_prices = encode_categorical(sell_prices, ["item_id", "store_id"] ).pipe(
 reduce_mem_usage 
)
<remove_duplicates>
%%time
cnt = 0
EPOCHS = 855
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
product = sales_train_val[['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']].drop_duplicates()<define_variables>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
nrows = 365 * 2 * NUM_ITEMS<define_variables>
sub['FVC1'] = 1.*pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
subm.loc[~subm.FVC1.isnull() ].head(10 )
d_name = ['d_' + str(i+1)for i in range(1913)]
sales_train_val_values = sales_train_val[d_name].values
tmp = np.tile(np.arange(1,1914),(sales_train_val_values.shape[0],1))
df_tmp =(( sales_train_val_values>0)* tmp)
 start_no = np.min(np.where(df_tmp==0,9999,df_tmp),axis=1)-1
flag = np.dot(np.diag(1/(start_no+1)) , tmp)<1
sales_train_val_values = np.where(flag,np.nan,sales_train_val_values)
 sales_train_val[d_name] = sales_train_val_values
del tmp,sales_train_val_values
gc.collect()<define_variables>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
1913-np.max(start_no )<categorify>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
sales_train_val = pd.melt(sales_train_val,
id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],
var_name = 'day', value_name = 'demand' )<split>
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission_regression.csv", index=False )
sales_train_val = sales_train_val.iloc[-nrows:,:]
sales_train_val = sales_train_val[~sales_train_val.demand.isnull() ]<rename_columns>
reg_sub = subm[["Patient_Week","FVC","Confidence"]].copy()
test1_rows = [row for row in submission['id'] if 'validation' in row]
test2_rows = [row for row in submission['id'] if 'evaluation' in row]
test1 = submission[submission['id'].isin(test1_rows)]
test2 = submission[submission['id'].isin(test2_rows)]
test1.columns = ["id"] + [f"d_{d}" for d in range(1914, 1914 + DAYS_PRED)]
test2.columns = ["id"] + [f"d_{d}" for d in range(1942, 1942 + DAYS_PRED)]
test2['id'] = test2['id'].str.replace('_evaluation','_validation')
 test1 = test1.merge(product, how = 'left', on = 'id')
 test2 = test2.merge(product, how = 'left', on = 'id')
 test1 = pd.melt(test1, id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],
var_name = 'day', value_name = 'demand')
 test2 = pd.melt(test2, id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],
var_name = 'day', value_name = 'demand')
 sales_train_val['part'] = 'train'
test1['part'] = 'test1'
test2['part'] = 'test2'
data = pd.concat([sales_train_val, test1, test2], axis = 0)
 del sales_train_val, test1, test2
data = data[data['part'] != 'test2']
gc.collect()<merge>
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True )
calendar.drop(['weekday', 'wday', 'month', 'year'],
inplace = True, axis = 1)
 data = pd.merge(data, calendar, how = 'left', left_on = ['day'], right_on = ['d'])
 data.drop(['d', 'day'], inplace = True, axis = 1)
 del calendar
gc.collect()
data = data.merge(sell_prices, on = ['store_id', 'item_id', 'wm_yr_wk'], how = 'left')
 print('Our final dataset to train has {} rows and {} columns'.format(data.shape[0], data.shape[1]))
del sell_prices
gc.collect()<categorify>
df = df1[['Patient_Week']].copy()
df['FVC'] = FVC_weight*df1['FVC'] +(1-FVC_weight)*df2['FVC']
df['Confidence'] = Confidence_weight*df1['Confidence'] +(1-Confidence_weight)*df2['Confidence']
df.head()
<prepare_x_and_y><EOS> 
df.to_csv('submission.csv', index=False )