Kaggle competition: Categorical Feature Encoding Challenge
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )<feature_engineering>
train = pd.read_csv('.. /input/cat-in-the-dat/train.csv', index_col='id')
 test = pd.read_csv('.. /input/cat-in-the-dat/test.csv', index_col='id' )
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 FE += ['age','week','BASE']<filter>
train = train.drop('bin_0', axis=1)
 test = test.drop('bin_0', axis=1 )
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data<compute_test_metric>
map_bin_3_4 = {'T': 1, 'F': 0, 'Y': 1, 'N': 0}
train['bin_3'] = train['bin_3'].map(map_bin_3_4)
 test['bin_3'] = test['bin_3'].map(map_bin_3_4)
 train['bin_4'] = train['bin_4'].map(map_bin_3_4)
 test['bin_4'] = test['bin_4'].map(map_bin_3_4 )
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model<prepare_x_and_y>
train['ord_5_1'] = train['ord_5'].str[0]
train['ord_5_2'] = train['ord_5'].str[1]
train = train.drop('ord_5', axis=1)
 test['ord_5_1'] = test['ord_5'].str[0]
test['ord_5_2'] = test['ord_5'].str[1]
test = test.drop('ord_5', axis=1 )
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
nh = z.shape[1]
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))<choose_model_class>
train = train.drop('ord_5_2', axis=1)
 test = test.drop('ord_5_2', axis=1 )
NFOLD = 2
kf = KFold(n_splits=NFOLD )<predict_on_test>
ord_1_values = ['Novice', 'Contributor', 'Expert', 'Master', 'Grandmaster']
map_ord_1 = lambda x: ord_1_values.index(x)
 train['ord_1'] = train['ord_1'].apply(map_ord_1)
 test['ord_1'] = test['ord_1'].apply(map_ord_1 )
%%time
cnt = 0
EPOCHS = 600
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD<compute_test_metric>
ord_2_values = ['Freezing', 'Cold', 'Warm', 'Hot', 'Boiling Hot', 'Lava Hot']
map_ord_2 = lambda x: ord_2_values.index(x)
 train['ord_2'] = train['ord_2'].apply(map_ord_2)
 test['ord_2'] = test['ord_2'].apply(map_ord_2 )
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )<prepare_output>
map_to_ascii_index = lambda x: string.ascii_letters.index(x)
 for column in ['ord_3', 'ord_4', 'ord_5_1']:
train[column] = train[column].apply(map_to_ascii_index)
 test[column] = test[column].apply(map_to_ascii_index )
sub['FVC1'] = 1.*pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
subm.loc[~subm.FVC1.isnull() ].head(10 )<feature_engineering>
columns_to_test = ['nom_7', 'nom_8', 'nom_9']
replace_xor = lambda x: 'xor' if x in xor_values else x
for column in columns_to_test:
xor_values = set(train[column].unique())^ set(test[column].unique())
 if xor_values:
print('Column', column, 'has', len(xor_values), 'XOR values')
 train[column] = train[column].apply(replace_xor)
 test[column] = test[column].apply(replace_xor)
 else:
print('Column', column, 'has no XOR values' )
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
sigma_mean = 60
if sigma_mean<sigma_mean:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']<load_from_csv>
def get_infrequent_values(data, column, threshold):
value_counts = data[column].value_counts()
return list(value_counts[value_counts < threshold].index )
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1<save_to_csv>
for column in train.columns:
n = len(get_infrequent_values(train, column, 3))
if n > 0:
print('Column', column, 'has', n, 'unique infrequent value(s)' )
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission_regression.csv", index=False )<prepare_output>
thresholds = {
'nom_9': 3
}
for column in thresholds.keys() :
values_to_replace = get_infrequent_values(train, column, thresholds[column])
 replace = lambda x: 'value' if x in values_to_replace else x
train[column] = train[column].transform(replace)
 test[column] = test[column].transform(replace )
reg_sub = subm[["Patient_Week","FVC","Confidence"]].copy()<sort_values>
for column in thresholds.keys() :
print(column, ':', train[column].value_counts() ['value'], 'values replaced in train dataset' )
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True )<feature_engineering>
day_map = {
1: 3,
2: 2,
3: 1,
4: 0,
5: 1,
6: 2,
7: 3
}
train['day'] = train['day'].map(day_map)
 test['day'] = test['day'].map(day_map )
df = df1[['Patient_Week']].copy()
df['FVC'] =(0.45*df1['FVC'] + 0.55*df2['FVC'])
 df['Confidence'] =(0.45*df1['Confidence'] + 0.55*df2['Confidence'])
 df.head()<save_to_csv>
features_to_drop_most_frequent_values = ['nom_1', 'nom_3', "day"]
most_frequent_values = [train[column].value_counts().index[0] for column in features_to_drop_most_frequent_values]
most_frequent_values
df.to_csv('submission.csv', index=False )<set_options>
thermometer_features = ['ord_1', 'ord_2', 'ord_3', 'ord_4', 'month', 'ord_5_1']
ohe_2_features = ['nom_1', 'nom_3', 'day']
ohe_3_features = ['nom_8']
ohe_1_features = set(test.columns)- set(ohe_2_features)- set(ohe_3_features)- set(thermometer_features )
warnings.simplefilter('ignore')
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 quantiles = [0.2, 0.5, 0.8]<categorify>
y_train = train['target'].copy()
x_train = train.drop('target', axis=1)
 del train
x_test = test.copy()
del test
def make_X(dt, dense_cols, cat_feats):
X = {"dense": dt[dense_cols].to_numpy() }
for i, v in enumerate(cat_feats):
X[v] = dt[[v]].to_numpy()
return X
class Loader:
def __init__(self, X, y, shuffle=True, batch_size=64, cat_cols=[]):
self.X_cont = X["dense"]
self.X_cat = np.concatenate([X[k] for k in cat_cols], axis=1)
 self.y = y
self.shuffle = shuffle
self.batch_size = batch_size
self.n_conts = self.X_cont.shape[1]
self.len = self.X_cont.shape[0]
n_batches, remainder = divmod(self.len, self.batch_size)
 if remainder > 0:
n_batches += 1
self.n_batches = n_batches
self.remainder = remainder
self.idxes = np.array([i for i in range(self.len)])
 def __iter__(self):
self.i = 0
if self.shuffle:
ridxes = self.idxes
np.random.shuffle(ridxes)
 self.X_cat = self.X_cat[[ridxes]]
self.X_cont = self.X_cont[[ridxes]]
if self.y is not None:
self.y = self.y[[ridxes]]
return self
def __next__(self):
if self.i >= self.len:
raise StopIteration
if self.y is not None:
y = torch.FloatTensor(self.y[self.i:self.i + self.batch_size].astype(np.float32))
else:
y = None
xcont = torch.FloatTensor(self.X_cont[self.i:self.i + self.batch_size])
 xcat = torch.LongTensor(self.X_cat[self.i:self.i + self.batch_size])
 batch =(xcont, xcat, y)
 self.i += self.batch_size
return batch
def __len__(self):
return self.n_batches<choose_model_class>
class ThermometerEncoder(BaseEstimator, TransformerMixin):
def __init__(self, cols=None, drop_invariant=True):
self.cols = cols
self.drop_invariant = drop_invariant
def get_params(self, deep=True):
return {'cols': self.cols, 'drop_invariant': self.drop_invariant}
def set_params(self, **parameters):
for parameter, value in parameters.items() :
setattr(self, parameter, value)
 return self
def fit(self, X, y=None):
self.bars = {}
for c in self.cols:
k = np.arange(X[c].max() + 1)
 self.bars[c] =(k[:-1] < k.reshape(-1, 1)).astype(int)
 return self
def transform(self, X, y=None):
out = pd.DataFrame(index=X.index)
 for c in self.cols:
out = out.join(self.transform_one(X, c))
if self.drop_invariant:
columns_to_drop = []
for c in out.columns:
if len(out[c].unique())== 1:
columns_to_drop.append(c)
 out = out.drop(columns_to_drop, axis=1)
 return out
def transform_one(self, X, c):
bars = self.bars[c]
out = pd.DataFrame(index=X.index, data=bars[X[c]])
 out.columns = [c + '_' + str(k)for k in range(bars.shape[1])]
return out
class model_nn(nn.Module):
def __init__(self, hidden_dim, output_dim, emb_dims, n_cont):
super().__init__()
self.emb_layers = nn.ModuleList([nn.Embedding(x, y)for x, y in emb_dims])
 n_embs = sum([y for x, y in emb_dims])
 self.n_embs = n_embs
self.n_cont = n_cont
inp_dim = n_embs + n_cont
self.inp_dim = inp_dim
self.fs0 = nn.Linear(inp_dim, hidden_dim)
 self.relufs0 = nn.ELU()
self.fs1 = nn.Linear(hidden_dim, inp_dim)
 self.fs2 = nn.Sigmoid()
self.fc0 = nn.Linear(inp_dim, hidden_dim)
 self.relu0 = nn.ELU()
self.fc1 = nn.Linear(hidden_dim, hidden_dim)
 self.relu1 = nn.ELU()
self.fc2 = nn.Linear(hidden_dim, output_dim)
 self.fc3 = nn.Linear(hidden_dim, output_dim)
 self.fc4 = nn.Linear(hidden_dim, output_dim)
 def encode_and_combine_data(self, cat_data):
xcat = [el(cat_data[:, k])for k, el in enumerate(self.emb_layers)]
xcat = torch.cat(xcat, 1)
 return xcat
def forward(self, cont_data, cat_data):
cont_data = cont_data.to(device)
 cat_data = cat_data.to(device)
 cat_data = self.encode_and_combine_data(cat_data)
 x = torch.cat([cont_data, cat_data], dim=1)
 w = self.fs0(x)
 w = self.relufs0(w)
 w = self.fs1(w)
 w = self.fs2(w)
 wx = w*x + x
hz = self.fc0(wx)
 hz = self.relu0(hz)
 hz = self.fc1(hz)
 hz = self.relu1(hz)
 out1 = self.fc2(hz)
 out2 = self.fc3(hz)
 out3 = self.fc4(hz)
 return torch.cat([out1, out2, out3], dim=1 )<compute_test_metric>
train_part = len(x_train)
 traintest = pd.concat([x_train, x_test] )
def quantile_loss(preds, target, quantiles):
assert not target.requires_grad
assert preds.size(0)== target.size(0)
 losses = []
for i, q in enumerate(quantiles):
errors = target - preds[:, i]
losses.append(torch.max(( q - 1)* errors, q * errors ).unsqueeze(1))
loss = torch.mean(torch.sum(torch.cat(losses, dim=1), dim=1))
return loss
def metric(outputs, target):
confidence = np.abs(outputs[:, 2] - outputs[:, 0])
 clip = np.where(confidence > 70, confidence, 70)
 delta = np.abs(outputs[:, 1] - target)
 delta = np.where(delta > 1000, 1000, delta)
 metrics =(delta*np.sqrt(2)/clip)+ np.log(clip*np.sqrt(2))
return np.mean(metrics )<define_variables>
from sklearn.preprocessing import OneHotEncoder
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
Model_Root = 'models'<load_from_csv>
ohe_1 = OneHotEncoder(dtype='uint8', handle_unknown="ignore")
 ohe_1_traintest = ohe_1.fit_transform(traintest[ohe_1_features] )
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient")
 tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub])
 data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min')
 base = data.loc[data.Weeks == data.min_week]
base = base[['Patient', 'FVC', 'Percent']].copy()
base.columns = ['Patient','min_FVC', 'min_percent']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True)
 data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
 data['age_week'] = data['Age'].values + data['base_week'].values/53
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['min_percent_norm'] =(data['min_percent'] - data['min_percent'].min())/(data['min_percent'].max() - data['min_percent'].min())
 data['age_week_norm'] =(data['age_week'] - data['age_week'].min())/(data['age_week'].max() - data['age_week'].min())
 cat_feat = ['Sex', 'SmokingStatus']
uniques = []
for i, v in enumerate(cat_feat):
data[v] = OrdinalEncoder(dtype="int" ).fit_transform(data[[v]])
 uniques.append(len(data[v].unique()))
tr = data.loc[data.WHERE == 'train']
chunk = data.loc[data.WHERE == 'val']
sub = data.loc[data.WHERE == 'test']
FE += ['age', 'week', 'BASE', 'min_percent_norm', 'age_week_norm']
print(list(tr))
print(list(sub))
print(FE)
 print(cat_feat)
 print(uniques)
 dims = [1, 2]
emb_dims = [(x, y)for x, y in zip(uniques, dims)]
n_cont = len(FE )<train_model>
ohe_2 = OneHotEncoder(drop=most_frequent_values, dtype='uint8')
 ohe_2_traintest = ohe_2.fit_transform(traintest[ohe_2_features] )
hidden_dim = 256
out_dim = 1
kfold = 6
groups = np.asarray(tr['Patient'].values)
 skf = GroupKFold(n_splits=kfold)
 avg_preds = np.zeros(( len(sub), len(quantiles)))
 models = []
for i,(train_index, test_index)in enumerate(skf.split(tr, tr['FVC'].values, groups=groups)) :
print('[Fold %d/%d]' %(i + 1, kfold))
model_path = f"{Model_Root}/nn_model_%s.pth" % i
if os.path.isfile(model_path):
final_model = torch.load(model_path)
 X_test = make_X(sub, FE, cat_feat)
 test_loader = Loader(X_test, None, cat_cols=cat_feat, batch_size=256, shuffle=False)
 preds = []
for i,(X_cont, X_cat, y)in enumerate(tqdm(test_loader)) :
print(X_cont.shape, X_cat.shape)
 out = final_model(X_cont, X_cat)
 preds.append(out)
 preds = torch.cat(preds, dim=0 ).detach().cpu().numpy()
avg_preds += preds
else:
X_train, X_valid = tr.iloc[train_index], tr.iloc[test_index]
y_train, y_valid = tr.iloc[train_index]['FVC'].values, tr.iloc[test_index]['FVC'].values
X_train = make_X(X_train.reset_index() , FE, cat_feat)
 X_valid = make_X(X_valid.reset_index() , FE, cat_feat)
 train_loader = Loader(X_train, y_train, cat_cols=cat_feat, batch_size=16, shuffle=True)
 val_loader = Loader(X_valid, y_valid, cat_cols=cat_feat, batch_size=64, shuffle=True)
 model = model_nn(hidden_dim, out_dim, emb_dims, n_cont ).to(device)
 final_model = model_training(model, train_loader, val_loader, epochs=1000,
batch_size=64, lr=0.01, patience=20,
model_path='nn_model_%s.pth' % i)
 models.append(final_model)
 for model in models:
X_test = make_X(sub, FE, cat_feat)
 test_loader = Loader(X_test, None, cat_cols=cat_feat, batch_size=256, shuffle=False)
 preds = []
with torch.no_grad() :
for i,(X_cont, X_cat, y)in enumerate(tqdm(test_loader)) :
out = model(X_cont, X_cat)
 preds.append(out)
 preds = torch.cat(preds, dim=0 ).detach().cpu().numpy()
avg_preds += preds
print(avg_preds)
 avg_preds = avg_preds/kfold
print(avg_preds)
 sub['Patient_Week'] = sub['Patient_Week'].values
sub['FVC'] = avg_preds[:, 1]
sub['Confidence'] = avg_preds[:, 2]-avg_preds[:, 0]
sub[['Patient_Week', 'FVC', 'Confidence']].to_csv("submission.csv", index=False )<set_options>
ohe_3 = OneHotEncoder(drop='first', dtype='uint8')
 ohe_3_traintest = ohe_3.fit_transform(traintest[ohe_3_features] )
cross_val_score, GroupKFold, GridSearchCV,
cross_val_predict, RandomizedSearchCV)
 plt.style.use("dark_background")
 %matplotlib inline<load_from_csv>
ohe_traintest = scipy.sparse.hstack([ohe_1_traintest, ohe_2_traintest, ohe_3_traintest] ).tocsr()
del ohe_1_traintest
del ohe_2_traintest
del ohe_3_traintest
train_files = tf.io.gfile.glob(main_dir+"/train/*/*")
 test_files = tf.io.gfile.glob(main_dir+"/test/*/*")
 sample_sub = pd.read_csv(main_dir + "/sample_submission.csv")
 train = pd.read_csv(main_dir + "/train.csv")
 test = pd.read_csv(main_dir + "/test.csv")
 print("Number of train patients: {}
Number of test patients: {:4}" 
.format(train.Patient.nunique() , test.Patient.nunique()))
print("
Total number of Train patient records: {}
Total number of Test patient records: {:6}" 
.format(len(train_files), len(test_files)))
 train.shape, test.shape, sample_sub.shape<compute_test_metric>
thermometer = ThermometerEncoder(thermometer_features)
 thermometer_traintest = thermometer.fit_transform(traintest )
def laplace_log_likelihood(y_true, y_pred, sigma=70):
sigma_clipped = tf.maximum(sigma, 70)
 delta_clipped = tf.minimum(tf.abs(y_true - y_pred), 1000)
 delta_clipped = tf.cast(delta_clipped, dtype=tf.float32)
 sigma_clipped = tf.cast(sigma_clipped, dtype=tf.float32)
 score = - tf.sqrt(2.0)* delta_clipped / sigma_clipped - tf.math.log(tf.sqrt(2.0)* sigma_clipped)
 return tf.reduce_mean(score )<compute_test_metric>
final_traintest = scipy.sparse.hstack([ohe_traintest, thermometer_traintest] ).tocsr()
def l1(s):
def scorer_func(x, y, sigma=s):
return laplace_log_likelihood(x, y, sigma=s ).numpy()
return make_scorer(scorer_func, greater_is_better=False )<merge>
final_x_train = final_traintest[:train_part]
final_x_test = final_traintest[train_part:]
def base_shift(data, q=50):
x = data.copy()
temp =(x.groupby("Patient")
.apply(lambda x: x.loc[int(
 np.percentile(x['Weeks'].index, q=q)
), ["Weeks", "FVC", "Percent"]]))
temp.rename(
 {"Weeks": "Base_Week",
"FVC": "Base_FVC",
"Percent": "Base_Percent"},
axis=1, inplace=True)
 x = x.merge(temp, on='Patient')
 x['Week_Offset'] = x['Weeks'] - x['Base_Week']
return x<merge>
del x_train
del x_test
def multi_baseweek_frame(data, display=True):

 op = data.merge(
 data[['Patient', 'Weeks', 'FVC', 'Percent']].rename(
 {"Weeks": "Base_Week",
"FVC": "Base_FVC",
"Percent": "Base_Percent"}, axis=1),
on='Patient')
 op['Week_Offset'] = op['Weeks'] - op['Base_Week']
op = op[op['Week_Offset'] != 0]
if display:
print("Number of Samples:{:5} -> {:5}
Number of Columns:{:5} -> {:5}".format(
 data.shape[0], op.shape[0], data.shape[1], op.shape[1]))
return op.sort_values(by=['Patient', 'Base_Week'] ).reset_index(drop=True )<categorify>
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import GridSearchCV
def augment_train_cosine(data, n_similar=3, threshold=0.25, display_sample=True):

 temp = base_shift(data.copy() , q=0)
 temp['present_minus_past'] = temp.groupby("Patient")['FVC'].transform('diff' ).fillna(0)
 temp['Week_diff'] = temp.groupby("Patient")['Weeks'].transform('diff' ).fillna(0)
 temp['pms'] =(temp['present_minus_past'] / temp['Week_diff'] ).replace([np.inf, -np.inf] ).fillna(0)
 temp['pms_min'] = temp.groupby("Patient")['pms'].transform('min')
 temp['pms_25'] = temp.groupby("Patient")['pms'].transform(lambda x: np.percentile(x, q=25))
temp['pms_mean'] = temp.groupby("Patient")['pms'].transform('mean')
 temp['pms_75'] = temp.groupby("Patient")['pms'].transform(lambda x: np.percentile(x, q=75))
temp['pms_max'] = temp.groupby("Patient")['pms'].transform('max')
 temp['pms_sum'] = temp.groupby("Patient")['pms'].transform('sum')
 temp['pmb_avg'] =(temp['Percent'] - temp['Base_Percent'] ).groupby(temp.Patient ).transform("mean")
 temp['p_std'] = temp.groupby("Patient")['Percent'].transform('std')
 temp = temp.merge(
 pd.concat([
temp.groupby("Patient" ).apply(
 lambda x:(x['Percent'].values[-1] - x['Percent'].values[0])/ 
(x['Weeks'].values[-1] - x['Weeks'].values[0])).rename("Slope"),
temp.groupby("Patient" ).apply(
 lambda x: x['Week_Offset'].iloc[np.argmax(x['pms'])] ).rename("pmsw_max"),
temp.groupby("Patient" ).apply(
 lambda x: x['Week_Offset'].iloc[np.argmin(x['pms'])] ).rename("pmsw_min")
 ], axis=1, ignore_index=False), on='Patient')
 temp = temp.groupby('Patient' ).head(1 ).reset_index(drop=True)
 temp['factor'] = temp['Base_FVC'] / temp['Base_Percent']
train_cols = [
'Patient',
'pms_25', 'pms_mean', 'pms_75',
'pms_sum', 'pmsw_min', 'pmsw_max',
'pmb_avg', 'Slope', 'factor'
]
cat_cols = np.intersect1d(['Sex', 'SmokingStatus'], train_cols)
 temp = temp[train_cols]
temp = pd.get_dummies(temp, columns=cat_cols, drop_first=True, prefix='', prefix_sep='')
 n_similar += 1
groups =(pd.DataFrame(
 np.argsort(
 cosine_similarity(temp.drop("Patient", 1), temp.drop('Patient', 1)))
 [:, -1:-n_similar-1:-1]))
groups = groups[~pd.DataFrame(np.sort(groups.values, axis=1)).duplicated() ]
groups = groups.applymap(lambda x: temp.Patient.to_dict() [x] ).apply(list, axis=1 ).to_dict()
aug_data = []
for group in tqdm(groups.values() , disable=not display_sample):
temp = base_shift(train[train.Patient.isin(group)], q=0)
 temp['base_per_diff_from_mean'] = temp['Base_Percent'] - temp['Base_Percent'].unique().mean()
temp['Percent_shifted'] = temp['Percent'] - temp['base_per_diff_from_mean']
temp['base_week_diff_from_mean'] = temp['Base_Week'] - temp['Base_Week'].unique().mean()
temp['Week_shifted'] = temp['Weeks'] - temp['base_week_diff_from_mean']
temp = pd.merge_ordered(
 temp.drop(['Age', 'Sex', 'SmokingStatus', 'Base_Week', 'Week_Offset'], axis=1), 
(temp.groupby("Week_shifted")['Percent_shifted'] 
.agg(['mean', 'count'] ).query(f'count > {n_similar * threshold}')
.drop("count", 1)) ,
on='Week_shifted', left_by='Patient' 
)
 aug_data.append(temp)
 temp = pd.concat(aug_data ).reset_index(drop=True)
 temp[['base_per_diff_from_mean', 'base_week_diff_from_mean', 'Base_Percent', 'Base_FVC']] =(
 temp.groupby("Patient")[['base_per_diff_from_mean', 'base_week_diff_from_mean',
'Base_Percent', 'Base_FVC']].fillna(method='ffill'))
temp['Week_aug'] = temp['Week_shifted'] + temp['base_week_diff_from_mean']
temp['Percent_aug'] = np.where(
 temp['Percent'].isna() ,
temp['mean'] + temp['base_per_diff_from_mean'],
temp['Percent'])
 temp = temp[~temp['Percent_aug'].isna() ]
test_ids = temp['FVC'].isna()
temp.loc[test_ids, 'FVC'] = LinearRegression().fit(
 temp.loc[~test_ids, ['Base_Percent', 'Percent_aug', 'Base_FVC']],
temp.loc[~test_ids, 'FVC'] ).predict(
 temp.loc[test_ids, ['Base_Percent', 'Percent_aug', 'Base_FVC']])
 temp = temp.groupby(["Patient", 'Week_aug'] ).mean().reset_index()
temp = temp[['Patient', 'Week_aug', 'Percent_aug', 'FVC']]
temp = temp.rename({'Week_aug': 'Weeks', 'Percent_aug': 'Percent'}, axis=1)
 temp['Weeks'] = temp['Weeks'].astype(int)
 temp = temp.merge(
(data[['Patient', 'Sex', 'Age', 'SmokingStatus']] 
.groupby('Patient' ).head(1 ).reset_index(drop=True)) ,
on='Patient')
 if display_sample:
print("Data augmented by factor: {:.2f}x".format(1 +(
 temp.shape[0] - data.shape[0])/ data.shape[0]))
f, ax = plt.subplots(nrows=4, ncols=2, figsize=(20, 20))
for i, pat in enumerate(temp.Patient.unique() [:4]):
ax[i][0].plot(*list(zip(*data[data.Patient == pat][['FVC', 'Weeks']].values))
[::-1], c='g', alpha=0.7)
 ax[i][1].plot(*list(zip(*data[data.Patient == pat][['Percent', 'Weeks']].values))
[::-1], c='g', alpha=0.7)
 ax[i][0].scatter(*list(zip(*temp[temp.Patient == pat][['FVC', 'Weeks']].values))
[::-1], c='r')
 ax[i][1].scatter(*list(zip(*temp[temp.Patient == pat][['Percent', 'Weeks']].values))
[::-1], c='r')
 ax[i][0].set(xlabel='Weeks', ylabel='FVC')
 ax[i][1].set(xlabel='Weeks', ylabel='Percent')
 f.suptitle("FVC & Percent Augmentation", y=.9)
 return temp<merge>
logit_param_grid = {
'C': [0.100, 0.150, 0.120, 0.125, 0.130, 0.135, 0.140, 0.145, 0.150]
}
logit_grid = GridSearchCV(LogisticRegression(solver='lbfgs'), logit_param_grid,
scoring='roc_auc', cv=5, n_jobs=-1, verbose=0)
 best_C = 0.14
print('Best C:', best_C )
def augment_train_naive(
 data, steps=5, method='index', noise=.25, val_split=0.25,
end_pts=[None, None], display_sample=True):

 temp = data[['Patient','Weeks', 'FVC', 'Percent']].merge(
(( data.groupby("Patient")['Weeks'] 
.apply(lambda x: pd.Series(
 np.union1d(np.arange(
 end_pts[0] if end_pts[0] else x.min() ,
end_pts[1] if end_pts[1] else x.max() ,
step=steps), x))
).reset_index(level=0))),
on=['Patient', 'Weeks'], how='right')
 temp.loc[:, ['FVC', 'Percent']] =(
 temp.groupby("Patient")[['FVC', 'Percent']] 
.apply(lambda x:(
 x.interpolate(method=method, limit_direction='both')+ 
(x.std().values * np.random.uniform(-noise, noise, [len(x), 1])))))
 temp = temp.merge(
 data.groupby("Patient")[['Patient', 'Age', 'Sex', 'SmokingStatus']].head(1),
on='Patient')
 if display_sample:
f, ax = plt.subplots(nrows=4, ncols=2, figsize=(20, 20))
for i, pat in enumerate(temp.Patient.unique() [:4]):
ax[i][0].plot(*list(zip(*data[data.Patient == pat][['FVC', 'Weeks']].values))
[::-1], c='g', alpha=0.7)
 ax[i][1].plot(*list(zip(*data[data.Patient == pat][['Percent', 'Weeks']].values))
[::-1], c='g', alpha=0.7)
 ax[i][0].scatter(*list(zip(*temp[temp.Patient == pat][['FVC', 'Weeks']].values)) [::-1], c='r')
 ax[i][1].scatter(*list(zip(*temp[temp.Patient == pat][['Percent', 'Weeks']].values)) [::-1], c='r')
 ax[i][0].set(xlabel='Weeks', ylabel='FVC')
 ax[i][1].set(xlabel='Weeks', ylabel='Percent')
 f.suptitle("FVC & Percent Augmentation", y=.9)
 print("Data augmented by factor: {:.2f}x".format(1 +(
 temp.shape[0] - data.shape[0])/ data.shape[0]))
return temp<data_type_conversions>
logit = LogisticRegression(C=best_C, solver="lbfgs", max_iter=5000)
 logit.fit(final_x_train, y_train)
 y_full_train = logit.predict_proba(final_x_test)[:, 1]
sub = sample_sub.Patient_Week.str.extract("(ID\w+)_(\-?\d+)" ).rename({0: "Patient", 1: "Weeks"}, axis=1)
 sub['Weeks'] = sub['Weeks'].astype(int)
 sub = pd.merge(sub, test[['Patient', 'Sex', 'SmokingStatus']], on='Patient')
 sub["Patient_Week"] = sub.Patient + "_" + sub.Weeks.astype(str)
 sub.head()<choose_model_class>
submission = pd.read_csv('.. /input/cat-in-the-dat/sample_submission.csv', index_col='id' )
class GBR(RegressorMixin, BaseEstimator):
def __init__(self, alpha=.75, **params):
self.alpha = alpha
self.umodel = self._create_model(loss='quantile', q=self.alpha, **params)
 self.mmodel = self._create_model(loss='lad', **params)
 self.lmodel = self._create_model(loss='quantile', q=1-self.alpha, **params)
 def _create_model(self, loss, q=.75, **params):
model = GradientBoostingRegressor(
 init=LinearRegression() ,
criterion='friedman_mse',
n_estimators=50, max_depth=2,
loss=loss, alpha=q, **params)
 return model
def fit(self, x, y):
self.umodel.fit(x, y)
 self.mmodel.fit(x, y)
 self.lmodel.fit(x, y)
 return self
def predict(self, X):
return self.mmodel.predict(X)
 def predict_forecast(self, X, return_bounds=False):
preds = self.mmodel.predict(X)
 upper = self.umodel.predict(X)
 lower = self.lmodel.predict(X)
 if return_bounds:
return preds, upper, lower
else:
return preds,(upper - lower )<concatenate>
submission['target'] = y_full_train
submission.to_csv('logit-full-train.csv' )