Kaggle competition: Categorical Feature Encoding Challenge
input_path = '.. /input/osic-pulmonary-fibrosis-progression'
pretrained_path = '.. /input/osic-linear-decay-and-quant-reg-base/pretrained_weights'<categorify>
submission = pd.read_csv('.. /input/cat-in-the-dat/sample_submission.csv', index_col='id' )
def height_proxy(fvc_e, age, sex):
if sex == 'Female': h = fvc_e/(21.78-0.101*age)
 else: h = fvc_e/(27.63-0.112*age)
 return h
def process_init_week(df, train_df = False):
if train_df:
df['min_week'] = df.groupby('Patient')['Weeks'].transform('min')
 base = df.loc[df.Weeks == df.min_week][['Patient', 'FVC', 'Percent', 'Age', 'Sex']]
base['FVC_init_avg']= base.groupby('Patient')['FVC'].transform('mean' ).astype(int)
 base['Percent_init']= base.groupby('Patient')['Percent'].transform('mean')
 base = base[['Patient', 'FVC_init_avg', 'Percent_init', 'Age', 'Sex']].drop_duplicates()
base['FVC_expected'] =(base['FVC_init_avg'] /(base['Percent_init']/100))
base['Height_proxy'] = base.apply(lambda x: height_proxy(x.FVC_expected, x.Age, x.Sex), axis=1)
 base = base[['Patient', 'Height_proxy', 'FVC_init_avg', 'Percent_init']]
df = df.merge(base, on='Patient', how='left')
 df['init_week'] = df['Weeks'] - df['min_week']
return df<load_from_csv>
kernels = pd.read_csv('.. /input/cat-in-dat-kernels/kernels.csv', index_col='id' )
train = pd.read_csv(input_path + '/train.csv')
 train = process_init_week(train, train_df = True)
 train.drop_duplicates(keep='first', inplace=True, subset=['Patient','Weeks'])
 train.head(3 )<feature_engineering>
def make_filename(idx):
return glob.glob('.. /input/cat-in-dat-kernels/' + str(idx)+ '__submission__*.csv')[0]
def read_predictions(idx):
temp = pd.read_csv(make_filename(idx), index_col='id')
 temp.columns = [str(idx)]
return temp
predictions = pd.concat([read_predictions(idx)for idx in kernels.index], axis=1)
 predictions.shape
sub = pd.read_csv(input_path + '/sample_submission.csv')
 test = pd.read_csv(input_path + '/test.csv')
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
test = test.rename(columns={'Weeks': 'min_week'})
 sub = sub.merge(test, on='Patient')
 sub = process_init_week(sub, train_df = False)
 sub.head(3 )<normalization>
submission['target'] = predictions.mean(axis=1)
 submission.to_csv('stack-mean.csv' )
def scale_fn(var_name):
col = train[var_name]
return lambda x:(x - col.min())/(col.max() - col.min())
 scale_age = scale_fn('Age')
 scale_height = scale_fn('Height_proxy')
 scale_percent = scale_fn('Percent')
 scale_fvc = scale_fn('FVC_init_avg')
 scale_week = lambda x:(x -(-12)) /(133-(-12))
def transform_features(df):
df = df.assign(sex_code = np.where(df['Sex'] == 'Female', 1, 0))
df = df.assign(ex_smoker = np.where(df['SmokingStatus'] == 'Ex-smoker', 1, 0))
df = df.assign(never_smoked = np.where(df['SmokingStatus'] == 'Never smoked', 1, 0))
df = df.assign(current_smoker = np.where(df['SmokingStatus'] == 'Currently smokes', 1, 0))
df['has_smoked'] = df['ex_smoker'] + df['current_smoker']
df['age'] = df['Age'].map(scale_age)
 df['height'] = df['Height_proxy'].map(scale_height)
 df['percent'] = df['Percent'].map(scale_percent)
 df['percent_init'] = df['Percent_init'].map(scale_percent)
 df['week'] = df['Weeks'].map(scale_week)
 df['fvc_init'] = df['FVC_init_avg'].map(scale_fvc)
 return df<feature_engineering>
scores = kernels['score']
sum_scores = sum(scores)
 weights = [x / sum_scores for x in scores]
train = transform_features(train)
 train.reset_index(inplace=True, drop = True)
 train.head(3 )<feature_engineering>
submission['target'] = sum_predictions
submission.to_csv('stack-weighted-sum.csv' )
sub = transform_features(sub)
 sub.head(3 )<prepare_output>
N = 3
selected = kernels.sort_values('score', ascending=False ).head(N )
PREDICTIONS = sub[['Patient', 'Weeks', 'Patient_Week']].copy()
PREDICTIONS.head(5 )<load_from_csv>
filter_predictions = predictions.loc[:,selected.index.values.astype(str)]
LD_inference = pd.read_csv(pretrained_path + '/inference_linear_decay_2020Sep19.csv')
 LD_inference<compute_train_metric>
submission['target'] = filter_predictions.mean(axis=1)
 submission.to_csv('stack-filtered.csv' )