Kaggle competition: Categorical Feature Encoding Challenge
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge<compute_test_metric>
import pandas as pd
import numpy as np
from sklearn import preprocessing
import seaborn as sns
from sklearn.model_selection import GridSearchCV
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
from sklearn import metrics
from sklearn.linear_model import LogisticRegression
import matplotlib.pyplot as plt
import scipy
from sklearn import linear_model, datasets
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model<prepare_x_and_y>
train = pd.read_csv("/kaggle/input/cat-in-the-dat/train.csv", index_col='id')
 test = pd.read_csv("/kaggle/input/cat-in-the-dat/test.csv", index_col = 'id' )
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
nh = z.shape[1]
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))<choose_model_class>
print(train.isnull().sum().sum())
 print(test.isnull().sum().sum() )
NFOLD = 2
kf = KFold(n_splits=NFOLD )<predict_on_test>
test.nunique()
%%time
cnt = 0
EPOCHS = 600
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD<compute_test_metric>
train['bin_4'] = train['bin_4'].map({'Y': 1, 'N': 0})
 train['bin_3'] = train['bin_3'].map({'T': 1, 'F': 0})
 train['ord_1'] = train['ord_1'].map({'Novice': 0, 'Contributor': 1, 'Expert': 2, 'Master': 3,'Grandmaster': 4})
 train['ord_2'] = train['ord_2'].map({'Freezing': 0, 'Cold': 1, 'Warm': 2, 'Hot': 3, 'Boiling Hot': 4, 'Lava Hot': 5} )
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )<prepare_output>
test['bin_4'] = test['bin_4'].map({'Y': 1, 'N': 0})
 test['bin_3'] = test['bin_3'].map({'T': 1, 'F': 0})
 test['ord_1'] = test['ord_1'].map({'Novice': 0, 'Contributor': 1, 'Expert': 2, 'Master': 3,'Grandmaster': 4})
 test['ord_2'] = test['ord_2'].map({'Freezing': 0, 'Cold': 1, 'Warm': 2, 'Hot': 3, 'Boiling Hot': 4, 'Lava Hot': 5} )
sub['FVC1'] = 1.*pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
subm.loc[~subm.FVC1.isnull() ].head(10 )<feature_engineering>
l3 = train[train['target'] == 1]['ord_3'].value_counts() / train['ord_3'].value_counts()
l4 = train[train['target'] == 1]['ord_4'].value_counts() / train['ord_4'].value_counts()
l5 = train[train['target'] == 1]['ord_5'].value_counts() / train['ord_5'].value_counts()
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
sigma_mean = 60
if sigma_mean<sigma_mean:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']<load_from_csv>
def sorted_ord(col_name,ratio):
s_ratio = ratio.sort_values()
keys = list(s_ratio.keys())
 train[col_name] = train[col_name].apply(lambda x : keys.index(x))
test[col_name] = test[col_name].apply(lambda x : keys.index(x))
sorted_ord('ord_3',l3)
 sorted_ord('ord_4',l4)
 sorted_ord('ord_5',l5 )
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1<save_to_csv>
nom_col = ['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4','nom_5','nom_6','nom_7','nom_8','nom_9','day','month']
nom_train = train[nom_col].astype(str)
 nom_test = test[nom_col].astype(str)
 ohc_test = pd.get_dummies(nom_test, sparse = True)
 ohc_train = pd.get_dummies(nom_train, sparse = True )
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission_regression.csv", index=False )<prepare_output>
unique_test = list(set(ohc_test.columns)- set(ohc_train.columns))
unique_train = list(set(ohc_train.columns)- set(ohc_test.columns))
print("Unique test columns: ", len(unique_test))
print("Unique Train columns: ",len(unique_train))
print(len(ohc_test.columns))
ohc_test_final = ohc_test.drop(unique_test, axis = 1)
 print(len(ohc_test_final.columns))
print(len(ohc_train.columns))
ohc_train_final = ohc_train.drop(unique_train, axis = 1)
 print(len(ohc_train_final.columns))
reg_sub = subm[["Patient_Week","FVC","Confidence"]].copy()<sort_values>
dropped_cols = ['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4','nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9','day','month','target']
final_df = train.drop(dropped_cols, axis =1)
 target_train = train['target']
dropped_cols_test = ['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4','nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9','day','month']
test_df = test.drop(dropped_cols_test, axis=1)
 df = final_df.to_sparse()
df1 = test_df.to_sparse()
df
df=df.to_coo()
df=df.tocsr()
df1
df1=df1.to_coo()
df1=df1.tocsr()
final_dataset = scipy.sparse.hstack([df,ohc_train_final])
 final_test = scipy.sparse.hstack([df1,ohc_test_final])
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True )<feature_engineering>
C = [0.12]
lr = LogisticRegression()
max_iter = [10000]
hyperparameters = dict(C=C, max_iter = max_iter)
 clf = GridSearchCV(lr, hyperparameters, cv=10, verbose=0)
 best_model = clf.fit(final_dataset,target_train )
df = df1[['Patient_Week']].copy()
df['FVC'] =(0.45*df1['FVC'] + 0.55*df2['FVC'])
 df['Confidence'] =(0.45*df1['Confidence'] + 0.55*df2['Confidence'])
 df.head()<save_to_csv>
predictions = best_model.predict_proba(final_test )
<install_modules><EOS> 
samplesub = pd.read_csv("/kaggle/input/cat-in-the-dat/sample_submission.csv", index_col="id")
 output = pd.DataFrame({'Id': samplesub.index, 'target': predictions[:,-1]})
 output.to_csv('submission.csv', index=False )