Kaggle competition: Categorical Feature Encoding Challenge
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge<load_from_csv>
!pip install target_encoding
subs = []
for model in models:
q = 0.5
sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 A_test, B_test, P_test,W, FVC= {}, {}, {},{},{}
STD, WEEK = {}, {}
for p in test.Patient.unique() :
x = []
tab = []
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 1.1 and int(i[:-4])/ len(ldir)> -0.1:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/{i}'))
tab.append(get_tab(test.loc[test.Patient == p, :]))
if len(x)<= 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q)
 A_test[p] = a
B_test[p] = test.FVC.values[test.Patient == p] - a*test.Weeks.values[test.Patient == p]
P_test[p] = test.Percent.values[test.Patient == p]
WEEK[p] = test.Weeks.values[test.Patient == p]
for k in sub.Patient_Week.values:
p, w = k.split('_')
 w = int(w)
 fvc = A_test[p] * w + B_test[p]
sub.loc[sub.Patient_Week == k, 'FVC'] = fvc
sub.loc[sub.Patient_Week == k, 'Confidence'] =(
 P_test[p] - A_test[p] * abs(WEEK[p] - w)
)
 _sub = sub[["Patient_Week","FVC","Confidence"]].copy()
subs.append(_sub )<prepare_output>
import pandas as pd
from target_encoding import TargetEncoderClassifier, TargetEncoder
from sklearn.model_selection import StratifiedKFold, cross_val_score
from sklearn.preprocessing import LabelEncoder
from sklearn.linear_model import LinearRegression, LogisticRegression
N = len(subs)
 sub = subs[0].copy()
sub["FVC"] = 0
sub["Confidence"] = 0
for i in range(N):
sub["FVC"] += subs[0]["FVC"] *(1/N)
 sub["Confidence"] += subs[0]["Confidence"] *(1/N )<save_to_csv>
train=pd.read_csv(".. /input/cat-in-the-dat/train.csv")
 test=pd.read_csv(".. /input/cat-in-the-dat/test.csv")
 sample_submission = pd.read_csv('.. /input/cat-in-the-dat/sample_submission.csv')
 cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42 )
sub[["Patient_Week","FVC","Confidence"]].to_csv("submission_img.csv", index=False )<prepare_output>
len_uniques = []
for c in train.columns.drop(['id', 'target']):
le = LabelEncoder()
le.fit(pd.concat([train[c], test[c]]))
train[c] = le.transform(train[c])
 test[c] = le.transform(test[c])
 print(c, len(le.classes_))
len_uniques.append(len(le.classes_))
X = train.drop(['target', 'id'], axis=1)
 y = train['target']
img_sub = sub[["Patient_Week","FVC","Confidence"]].copy()<load_from_csv>
ALPHA = 75
MAX_UNIQUE = max(len_uniques)
 FEATURES_COUNT = X.shape[1]
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )<concatenate>
lin = LogisticRegression()
score = cross_val_score(lin, X_train, y, scoring='roc_auc', cv=cv)
 print(f'score: {score.mean() :.4}, std: {score.std() :.4}')
 lin.fit(X_train, y)
 pred_lin = lin.predict_proba(X_test)[:,1]
<count_unique_values><EOS> 
sample_submission['target'] = pred_enc + pred_lin
sample_submission.to_csv('submission.csv', index=False )