Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<prepare_x_and_y>
import numpy as np
import pandas as pd
from scipy import stats
import matplotlib.pyplot as plt
import seaborn as sns
X=train.drop(['SalePrice','SalePrice_Log1p','Classes'],axis=1)
 y=train.SalePrice_Log1p<train_model>
df_train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv')
 df_test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv' )
def overfit_reducer(df):
overfit = []
for i in df.columns:
counts = df[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(df)* 100 > 99.9:
overfit.append(i)
 overfit = list(overfit)
 return overfit
overfitted_features = overfit_reducer(X )<normalization>
training_sample_counts = df_train.rename(columns={'Weeks': 'Samples'} ).groupby('Patient' ).agg('count')['Samples']
{("-")* 41}
{training_sample_counts}' )
std_scaler=StandardScaler()
rbst_scaler=RobustScaler()
power_transformer=PowerTransformer()
X_std=std_scaler.fit_transform(X)
 X_rbst=rbst_scaler.fit_transform(X)
 X_pwr=power_transformer.fit_transform(X)
 test_std=std_scaler.transform(test)
 test_rbst=rbst_scaler.transform(test)
 test_pwr=power_transformer.transform(test )<train_model>
df_submission = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 df_submission.head()
X_train,X_test,y_train,y_test=train_test_split(X_std,y,test_size=0.002,random_state=52)
 print('X_train Shape :',X_train.shape)
 print('X_test Shape :',X_test.shape)
 print('y_train Shape :',y_train.shape)
 print('y_test Shape :',y_test.shape )<train_model>
warnings.filterwarnings("ignore" )
lgb_regressor=lgb.LGBMRegressor(objective='regression', num_leaves=5, learning_rate=0.035, n_estimators=2177, max_bin=50, bagging_fraction=0.65,bagging_freq=5, bagging_seed=7,
feature_fraction=0.201, feature_fraction_seed=7,n_jobs=-1)
 lgb_regressor.fit(X_train, y_train)
 y_head=lgb_regressor.predict(X_test)
 print('-'*10+'LGBM'+'-'*10)
 print('R square Accuracy: ',r2_score(y_test,y_head))
print('Mean Absolute Error Accuracy: ',mean_absolute_error(y_test,y_head))
print('Mean Squared Error Accuracy: ',mean_squared_error(y_test,y_head))<train_model>
def seed_everything(seed=777):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed )
gb_reg = GradientBoostingRegressor(n_estimators=1992, learning_rate=0.03005, max_depth=4, max_features='sqrt', min_samples_leaf=15, min_samples_split=14, loss='huber', random_state =42)
 gb_reg.fit(X_train, y_train)
 y_head=gb_reg.predict(X_test)
 print('-'*10+'GBR'+'-'*10)
 print('R square Accuracy: ',r2_score(y_test,y_head))
print('Mean Absolute Error Accuracy: ',mean_absolute_error(y_test,y_head))
print('Mean Squared Error Accuracy: ',mean_squared_error(y_test,y_head))<compute_train_metric>
OUTPUT_DICT = './'
ID = 'Patient_Week'
TARGET = 'FVC'
SEED = 777
seed_everything(seed=SEED)
 N_FOLD = 7
kfolds = KFold(n_splits=10, shuffle=True, random_state=42)
 alphas=[1e-9,1e-8,1e-7,1e-6]
ridgecv_reg= make_pipeline(RidgeCV(alphas=alphas, cv=kfolds))
ridgecv_reg.fit(X_train, y_train)
 y_head=ridgecv_reg.predict(X_test)
 print('-'*10+'RidgeCV'+'-'*10)
 print('R square Accuracy: ',r2_score(y_test,y_head))
print('Mean Absolute Error Accuracy: ',mean_absolute_error(y_test,y_head))
print('Mean Squared Error Accuracy: ',mean_squared_error(y_test,y_head))<compute_train_metric>
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv')
 otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv' )
kfolds = KFold(n_splits=8, shuffle=True, random_state=42)
 lassocv_reg= make_pipeline(LassoCV(alphas=alphas, cv=kfolds))
lassocv_reg.fit(X_train, y_train)
 y_head=lassocv_reg.predict(X_test)
 print('-'*10+'LassoCV'+'-'*10)
 print('R square Accuracy: ',r2_score(y_test,y_head))
print('Mean Absolute Error Accuracy: ',mean_absolute_error(y_test,y_head))
print('Mean Squared Error Accuracy: ',mean_squared_error(y_test,y_head))<compute_train_metric>
train = pd.concat([train,otest])
 output = pd.DataFrame()
gb = train.groupby('Patient')
 tk0 = tqdm(gb, total=len(gb))
for _, usr_df in tk0:
usr_output = pd.DataFrame()
for week, tmp in usr_df.groupby('Weeks'):
rename_cols = {'Weeks': 'base_Week', 'FVC': 'base_FVC', 'Age': 'base_Age'}
tmp = tmp.rename(columns=rename_cols)
 drop_cols = ['Age', 'Sex', 'SmokingStatus', 'Percent']
_usr_output = usr_df.drop(columns=drop_cols ).rename(columns={'Weeks': 'predict_Week'} ).merge(tmp, on='Patient')
 _usr_output['Week_passed'] = _usr_output['predict_Week'] - _usr_output['base_Week']
usr_output = pd.concat([usr_output, _usr_output])
 print(usr_output)
 output = pd.concat([output, usr_output])
 train = output[output['Week_passed']!=0].reset_index(drop=True )
kfolds = KFold(n_splits=8, shuffle=True, random_state=42)
 alphas=[0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006]
l1ratio=[0.87, 0.9,0.92, 0.95,0.97, 0.99, 1]
elasticv_reg= make_pipeline(ElasticNetCV(alphas=alphas, cv=kfolds, l1_ratio=l1ratio))
elasticv_reg.fit(X_train, y_train)
 y_head=elasticv_reg.predict(X_test)
 print('-'*10+'ElasticNetCV'+'-'*10)
 print('R square Accuracy: ',r2_score(y_test,y_head))
print('Mean Absolute Error Accuracy: ',mean_absolute_error(y_test,y_head))
print('Mean Squared Error Accuracy: ',mean_squared_error(y_test,y_head))<train_model>
test = otest.rename(columns={'Weeks': 'base_Week', 'FVC': 'base_FVC', 'Age': 'base_Age'})
 submission = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 submission['Patient'] = submission['Patient_Week'].apply(lambda x: x.split('_')[0])
 submission['predict_Week'] = submission['Patient_Week'].apply(lambda x: x.split('_')[1] ).astype(int)
 test = submission.drop(columns=['FVC', 'Confidence'] ).merge(test, on='Patient')
 test['Week_passed'] = test['predict_Week'] - test['base_Week']
test.set_index('Patient_Week', inplace=True )
estimators = [('lgbm', lgb_regressor), 
('gbr', gb_reg), 
('lasso', lassocv_reg), 
('ridge', ridgecv_reg), 
('elasticnet', elasticv_reg)]
stack_reg=StackingRegressor(estimators=estimators,final_estimator=ExtraTreesRegressor(n_estimators=50),n_jobs=-1)
 stack_reg.fit(X_train, y_train)
 y_head=stack_reg.predict(X_test)
 print('-'*10+'StackingRegressor'+'-'*10)
 print('R square Accuracy: ',r2_score(y_test,y_head))
print('Mean Absolute Error Accuracy: ',mean_absolute_error(y_test,y_head))
print('Mean Squared Error Accuracy: ',mean_squared_error(y_test,y_head))<concatenate>
OUTPUT_DICT = './'
ID = 'Patient_Week'
TARGET = 'FVC'
SEED = 777
seed_everything(seed=SEED)
 N_FOLD = 7
y_head=pd.DataFrame(y_head,columns=['Predict'])
 y_test.reset_index(drop=True,inplace=True)
 y_test_y_head=pd.concat([y_test,y_head],axis=1)
 y_test_y_head.head()<count_values>
folds = train[['Patient', TARGET]].copy()
Fold = GroupKFold(n_splits=N_FOLD)
 groups = folds['Patient'].values
for n,(train_index, val_index)in enumerate(Fold.split(folds, folds[TARGET], groups)) :
folds.loc[val_index, 'fold'] = int(n)
 folds['fold'] = folds['fold'].astype(int )
print('Count columns of test: ',len(test.columns))
print('Count columns of X: ',len(X.columns))<predict_on_test>
def run_single_model(clf, train_df, test_df, folds, features, target, fold_num=0):
trn_idx = folds[folds.fold!=fold_num].index
val_idx = folds[folds.fold==fold_num].index
y_tr = target.iloc[trn_idx].values
X_tr = train_df.iloc[trn_idx][features].values
y_val = target.iloc[val_idx].values
X_val = train_df.iloc[val_idx][features].values
oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
clf.fit(X_tr, y_tr)
 oof[val_idx] = clf.predict(X_val)
 predictions += clf.predict(test_df[features])
 return oof, predictions
def run_kfold_model(clf, train, test, folds, features, target, n_fold=7):
oof = np.zeros(len(train))
predictions = np.zeros(len(test))
feature_importance_df = pd.DataFrame()
for fold_ in range(n_fold):
_oof, _predictions = run_single_model(clf,
train,
test,
folds,
features,
target,
fold_num=fold_)
 oof += _oof
predictions += _predictions/n_fold
return oof, predictions
test_pred_lgb=lgb_regressor.predict(test_pwr)
 test_pred_gb=gb_reg.predict(test_pwr)
 test_pred_elastic=elasticv_reg.predict(test_pwr)
 test_pred_ridge=ridgecv_reg.predict(test_pwr)
 test_pred_lasso=lassocv_reg.predict(test_pwr)
 test_pred_stack=stack_reg.predict(test_pwr )<create_dataframe>
target = train[TARGET]
test[TARGET] = np.nan
cat_features = ['Sex', 'SmokingStatus']
num_features = [c for c in test.columns if(test.dtypes[c] != 'object')&(c not in cat_features)]
features = num_features + cat_features
drop_features = [TARGET, 'predict_Week', 'Percent', 'base_Week']
features = [c for c in features if c not in drop_features]
if cat_features:
ce_oe = ce.OrdinalEncoder(cols=cat_features, handle_unknown='impute')
 ce_oe.fit(train)
 train = ce_oe.transform(train)
 test = ce_oe.transform(test )
test_pred_lgb=pd.DataFrame(test_pred_lgb,columns=['SalePrice'])
 test_pred_gb=pd.DataFrame(test_pred_gb,columns=['SalePrice'])
 test_pred_elastic=pd.DataFrame(test_pred_elastic,columns=['SalePrice'])
 test_pred_ridge=pd.DataFrame(test_pred_ridge,columns=['SalePrice'])
 test_pred_lasso=pd.DataFrame(test_pred_lasso,columns=['SalePrice'])
 test_pred_stack=pd.DataFrame(test_pred_stack,columns=['SalePrice'] )<feature_engineering>
scorelist = []
for alpha1 in [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1]:
for l1s in [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1]:
print(" For alpha:",alpha1,"& l1_ratio:",l1s)
 clf = ElasticNet(alpha=alpha1, l1_ratio = l1s)
 oof, predictions = run_kfold_model(clf, train, test, folds, features, target, n_fold=N_FOLD)
 train['FVC_pred'] = oof
test['FVC_pred'] = predictions
train['Confidence'] = 100
train['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))
train['diff'] = abs(train['FVC'] - train['FVC_pred'])
 train['delta'] = train['diff'].apply(lambda x: min(x, 1000))
train['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])
 score = train['score'].mean()
print(score)
 def loss_func(weight, row):
confidence = weight
sigma_clipped = max(confidence, 70)
 diff = abs(row['FVC'] - row['FVC_pred'])
 delta = min(diff, 1000)
 score = -math.sqrt(2)*delta/sigma_clipped - np.log(math.sqrt(2)*sigma_clipped)
 return -score
results = []
tk0 = tqdm(train.iterrows() , total=len(train))
for _, row in tk0:
loss_partial = partial(loss_func, row=row)
 weight = [100]
result = sp.optimize.minimize(loss_partial, weight, method='SLSQP')
 x = result['x']
results.append(x[0])
 train['Confidence'] = results
train['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))
train['diff'] = abs(train['FVC'] - train['FVC_pred'])
 train['delta'] = train['diff'].apply(lambda x: min(x, 1000))
train['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])
 score = train['score'].mean()
scorelist.append(score)
 print(score )
test_pred_lgb.SalePrice =np.floor(np.expm1(test_pred_lgb.SalePrice))
test_pred_gb.SalePrice =np.floor(np.expm1(test_pred_gb.SalePrice))
test_pred_elastic.SalePrice =np.floor(np.expm1(test_pred_elastic.SalePrice))
test_pred_ridge.SalePrice =np.floor(np.expm1(test_pred_ridge.SalePrice))
test_pred_lasso.SalePrice =np.floor(np.expm1(test_pred_lasso.SalePrice))
test_pred_stack.SalePrice =np.floor(np.expm1(test_pred_stack.SalePrice))<prepare_output>
TARGET = 'Confidence'
target = train[TARGET]
test[TARGET] = np.nan
cat_features = ['Sex', 'SmokingStatus']
num_features = [c for c in test.columns if(test.dtypes[c] != 'object')&(c not in cat_features)]
features = num_features + cat_features
drop_features = [ID, TARGET, 'predict_Week', 'base_Week', 'FVC', 'FVC_pred']
features = [c for c in features if c not in drop_features]
oof, predictions = run_kfold_model(clf, train, test, folds, features, target, n_fold=N_FOLD )
final_pred=(test_pred_stack*0.1665)+(test_pred_lgb*0.678)+(test_pred_ridge*0.1665)
 final_pred.head()<prepare_output>
train['Confidence'] = oof
train['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))
train['diff'] = abs(train['FVC'] - train['FVC_pred'])
 train['delta'] = train['diff'].apply(lambda x: min(x, 1000))
train['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])
 score = train['score'].mean()
print(score )
sample_sub['SalePrice']=final_pred<save_to_csv>
test['Confidence'] = predictions
test = test.reset_index()
<install_modules><EOS> 
sub = submission[['Patient_Week']].merge(test[['Patient_Week', 'FVC_pred', 'Confidence']], on='Patient_Week')
 sub = sub.rename(columns={'FVC_pred': 'FVC'})
 for i in range(len(otest)) :
sub.loc[sub['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
sub.loc[sub['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
sub[sub.Confidence<1]
sub.to_csv('submission.csv', index=False, float_format='%.1f' )