Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<define_variables>
import tensorflow as tf
import pandas as pd
import numpy as np
import os
import random
import pickle
import time
from tensorflow import keras as K
from tensorflow.keras import layers as L
from sklearn.model_selection import KFold
from sklearn.metrics import mean_absolute_error
import matplotlib.pyplot as plt
all_diagnosis = []
for _, row in df.iterrows() :
id_code = row["id_code"]
if id_code in id_to_diagnosis:
all_diagnosis.append(id_to_diagnosis[id_code])
 else:
all_diagnosis.append(0 )<save_to_csv>
def seed_all(seed = 20):
os.environ['PYTHONHASHSEED']=str(seed)
 random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_all(20 )
id_codes = df.id_code.values
new_df = pd.DataFrame.from_dict(data={"id_code": df.id_code.values, "diagnosis": all_diagnosis})
 new_df.to_csv("submission.csv", index=False )<groupby>
train = pd.read_csv('/kaggle/input/data-preparation-for-osic/train.csv')
 raw_test = pd.read_csv('/kaggle/input/osic-pulmonary-fibrosis-progression/test.csv')
 X_prediction = pd.read_csv('/kaggle/input/osic-pulmonary-fibrosis-progression/sample_submission.csv' )
def rank_by_team(df):
cols_to_drop = ['Id', 'groupId', 'matchId', 'winPlacePerc', "matchType"]
features = [col for col in df.columns if col not in cols_to_drop]
agg = df.groupby(['matchId', 'groupId'])[features].mean()
agg = agg.groupby('matchId')[features].rank(pct=True)
 return df.merge(agg, suffixes=['', '_mean_rank'], how='left', on=['matchId', 'groupId'])
 df = pd.read_csv('.. /input/pubg-finish-placement-prediction/train_V2.csv')
 df = df.dropna()
df = rank_by_team(df)
 cols_to_drop = ['Id', 'groupId', 'matchId', 'winPlacePerc', "matchType"]
features = [col for col in df.columns if col not in cols_to_drop]
X = df.loc[:, features]
Y = df.loc[:, "winPlacePerc"]
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=1)
 X_train = X
Y_train = Y
model = LinearRegression()
model.fit(X_train, Y_train)
 test_data = pd.read_csv(".. /input/pubg-finish-placement-prediction/test_V2.csv")
 test_data = rank_by_team(test_data)
 X = test_data.loc[:, features]
pred = model.predict(X)
 id = pd.read_csv(".. /input/pubg-finish-placement-prediction/sample_submission_V2.csv")
 id = id.loc[:, "Id"]
df = pd.DataFrame({"Id": id, "winPlacePerc": pred})
 df.to_csv("submission.csv", index=False, sep=',')
<load_from_csv>
ID='Patient_Week'
PINBALL_QUANTILE = [0.255, 0.50, 0.745]
LAMBDA_LOSS = 0.585
EPOCH = [54, 55, 20, 60, 23]
BATCH_SIZE = 128
NFOLD = 5
df_sub = pd.read_csv("submission.csv")
 df_test = pd.read_csv(".. /input/pubg-finish-placement-prediction/test_V2.csv")
 df_sub = df_sub.merge(df_test[["Id", "matchId", "groupId", "maxPlace", "numGroups"]], on="Id", how="left")
 df_sub_group = df_sub.groupby(["matchId", "groupId"] ).first().reset_index()
df_sub_group["rank"] = df_sub_group.groupby(["matchId"])["winPlacePerc"].rank()
df_sub_group = df_sub_group.merge(
 df_sub_group.groupby("matchId")["rank"].max().to_frame("max_rank" ).reset_index() ,
on="matchId", how="left")
 df_sub_group["adjusted_perc"] =(df_sub_group["rank"] - 1)/(df_sub_group["numGroups"] - 1)
 df_sub = df_sub.merge(df_sub_group[["adjusted_perc", "matchId", "groupId"]], on=["matchId", "groupId"], how="left")
 df_sub["winPlacePerc"] = df_sub["adjusted_perc"]
df_sub.loc[df_sub.maxPlace == 0, "winPlacePerc"] = 0
df_sub.loc[df_sub.maxPlace == 1, "winPlacePerc"] = 1
subset = df_sub.loc[df_sub.maxPlace > 1]
gap = 1.0 /(subset.maxPlace.values - 1)
 new_perc = np.around(subset.winPlacePerc.values / gap)* gap
df_sub.loc[df_sub.maxPlace > 1, "winPlacePerc"] = new_perc
df_sub.loc[(df_sub.maxPlace > 1)&(df_sub.numGroups == 1), "winPlacePerc"] = 0
assert df_sub["winPlacePerc"].isnull().sum() == 0
df_sub[["Id", "winPlacePerc"]].to_csv("submission.csv", index=False)
<import_modules>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.backend.mean(metric)
 def qloss(y_true, y_pred):
qs = PINBALL_QUANTILE
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.backend.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt<load_from_csv>
def eval_score(y_true, y_pred):
y_true = tf.dtypes.cast(y_true, tf.float32)*(data_prep.fvc_max-data_prep.fvc_min)+data_prep.fvc_min
y_pred = tf.dtypes.cast(y_pred, tf.float32)*(data_prep.fvc_max-data_prep.fvc_min)+data_prep.fvc_min
sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return -K.backend.mean(metric )
develop_mode = False
if develop_mode:
df_train = reduce_mem_usage(pd.read_csv('.. /input/train_V2.csv', nrows=5000))
df_test = reduce_mem_usage(pd.read_csv('.. /input/test_V2.csv'))
else:
df_train = reduce_mem_usage(pd.read_csv('.. /input/train_V2.csv'))
df_test = reduce_mem_usage(pd.read_csv('.. /input/test_V2.csv'))<train_model>
SELECTED_COLUMNS = ['Weeks', 'Percent', 'Age', 'Sex', 'Min_week', 'Base_FVC','Base_week', '_Currently smokes', '_Ex-smoker', '_Never smoked']
def create_model(lambda_loss):
model_input = K.Input(shape=(len(SELECTED_COLUMNS),))
x = L.Dense(500, activation="selu", name='dense_to_freeze1' )(model_input)
 x = L.Dense(100, activation="selu", name='dense_to_freeze2' )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="selu", name="p2" )(x)
 FVC = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="FVC" )([p1, p2])
 model = K.Model(
 inputs=model_input,
outputs=[FVC], 
)
 model.compile(
 optimizer=K.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False),
loss=mloss(lambda_loss),
metrics=score, 
)
 return model
model = create_model(LAMBDA_LOSS)
 model.summary()
print('The sizes of the datasets are:')
 print('Training Dataset: ', df_train.shape)
 print('Testing Dataset: ', df_test.shape )<sort_values>
X_prediction['Patient'] = X_prediction['Patient_Week'].str.extract(r' (.*)_.*')
 X_prediction['Weeks'] = X_prediction['Patient_Week'].str.extract(r'.*_ (.*)' ).astype(int)
 X_prediction = X_prediction[['Patient', 'Weeks', 'Patient_Week']]
rename_cols = {'Weeks_y':'Min_week', 'Weeks_x': 'Weeks', 'FVC':'Base_FVC'}
X_prediction = X_prediction.merge(raw_test, how='left', left_on='Patient', right_on='Patient' ).rename(columns=rename_cols)[['Patient', 'Min_week', 'Base_FVC', 'Percent', 'Age', 'Sex', 'SmokingStatus', 'Weeks', 'Patient_Week']].reset_index(drop=True )
group_tmp = df_train[df_train['matchId']=='df014fbee741c6']['groupId'].value_counts().sort_values(ascending=False )<set_options>
X_prediction['Base_week'] = X_prediction['Weeks'] - X_prediction['Min_week']
warnings.filterwarnings('ignore' )<load_from_csv>
def BuildFeature(is_train=True):

 y = None
test_idx = None
if is_train:
print("Reading train.csv")
 df = pd.read_csv('.. /input/train_V2.csv')
 df = df[df['maxPlace'] > 1]
else:
print("Reading test.csv")
 df = pd.read_csv('.. /input/test_V2.csv')
 test_idx = df.Id
df = reduce_mem_usage(df)
 print("Delete Unuseful Columns")
 target = 'winPlacePerc'
features = list(df.columns)
 features.remove("Id")
 features.remove("matchId")
 features.remove("groupId")
 features.remove("matchType")
 if is_train:
print("Read Labels")
 y = np.array(df.groupby(['matchId','groupId'])[target].agg('mean'), dtype=np.float64)
 features.remove(target)
 print("Read Group mean features")
 agg = df.groupby(['matchId','groupId'])[features].agg('mean')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
if is_train:
df_out = agg.reset_index() [['matchId','groupId']]
else:
df_out = df[['matchId','groupId']]
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 print("Read Group max features")
 agg = df.groupby(['matchId','groupId'])[features].agg('max')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId', 'groupId'])
 print("Read Group min features")
 agg = df.groupby(['matchId','groupId'])[features].agg('min')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId', 'groupId'])
 print("Read Group size features")
 agg = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_out = df_out.merge(agg, how='left', on=['matchId', 'groupId'])
 print("Read Match mean features")
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
df_out = df_out.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 print("Read Match size features")
 agg = df.groupby(['matchId'] ).size().reset_index(name='match_size')
 df_out = df_out.merge(agg, how='left', on=['matchId'])
 df_out.drop(["matchId", "groupId"], axis=1, inplace=True)
 X = df_out
feature_names = list(df_out.columns)
 del df, df_out, agg, agg_rank
gc.collect()
return X, y, feature_names, test_idx<prepare_x_and_y>
class OneHotEncoder(SklearnOneHotEncoder):
def __init__(self, **kwargs):
super(OneHotEncoder, self ).__init__(**kwargs)
 self.fit_flag = False
def fit(self, X, **kwargs):
out = super().fit(X)
 self.fit_flag = True
return out
def transform(self, X, categories, index='', name='', **kwargs):
sparse_matrix = super(OneHotEncoder, self ).transform(X)
 new_columns = self.get_new_columns(X=X, name=name, categories=categories)
 d_out = pd.DataFrame(sparse_matrix.toarray() , columns=new_columns, index=index)
 return d_out
def fit_transform(self, X, categories, index, name, **kwargs):
self.fit(X)
 return self.transform(X, categories=categories, index=index, name=name)
 def get_new_columns(self, X, name, categories):
new_columns = []
for j in range(len(categories)) :
new_columns.append('{}_{}'.format(name, categories[j]))
return new_columns
def standardisation(x, u, s):
return(x-u)/s
def normalization(x, ma, mi):
return(x-mi)/(ma-mi)
 class data_preparation() :
def __init__(self, bool_normalization=True,bool_standard=False):
self.enc_sex = LabelEncoder()
self.enc_smok = LabelEncoder()
self.onehotenc_smok = OneHotEncoder()
self.standardisation = bool_standard
self.normalization = bool_normalization
def __call__(self, data_untransformed):
data = data_untransformed.copy(deep=True)
 try:
data['Sex'] = self.enc_sex.transform(data['Sex'].values)
 data['SmokingStatus'] = self.enc_smok.transform(data['SmokingStatus'].values)
 data = pd.concat([data.drop(columns=['SmokingStatus']), self.onehotenc_smok.transform(data['SmokingStatus'].values.reshape(-1,1), categories=self.enc_smok.classes_, name='', index=data.index ).astype(int)], axis=1)
 if self.standardisation:
data['Base_week'] = standardisation(data['Base_week'],self.base_week_mean,self.base_week_std)
 data['Base_FVC'] = standardisation(data['Base_FVC'],self.base_fvc_mean,self.base_fvc_std)
 data['Base_percent'] = standardisation(data['Base_percent'],self.base_percent_mean,self.base_percent_std)
 data['Age'] = standardisation(data['Age'],self.age_mean,self.age_std)
 data['Weeks'] = standardisation(data['Weeks'],self.weeks_mean,self.weeks_std)
 if self.normalization:
data['Base_week'] = normalization(data['Base_week'],self.base_week_max,self.base_week_min)
 data['Base_FVC'] = normalization(data['Base_FVC'],self.base_fvc_max,self.base_fvc_min)
 data['Percent'] = normalization(data['Percent'],self.base_percent_max,self.base_percent_min)
 data['Age'] = normalization(data['Age'],self.age_max,self.age_min)
 data['Weeks'] = normalization(data['Weeks'],self.weeks_max,self.weeks_min)
 data['Min_week'] = normalization(data['Min_week'],self.base_week_max,self.base_week_min)
 except NotFittedError:
data['Sex'] = self.enc_sex.fit_transform(data['Sex'].values)
 data['SmokingStatus'] = self.enc_smok.fit_transform(data['SmokingStatus'].values)
 data = pd.concat([data.drop(columns=['SmokingStatus']), self.onehotenc_smok.fit_transform(data['SmokingStatus'].values.reshape(-1,1), categories=self.enc_smok.classes_, name='', index=data.index ).astype(int)], axis=1)
 if self.standardisation:
self.base_week_mean = data['Base_week'].mean()
self.base_week_std = data['Base_week'].std()
data['Base_week'] = standardisation(data['Base_week'],self.base_week_mean,self.base_week_std)
 self.base_fvc_mean = data['Base_FVC'].mean()
self.base_fvc_std = data['Base_FVC'].std()
data['Base_FVC'] = standardisation(data['Base_FVC'],self.base_fvc_mean,self.base_fvc_std)
 self.base_percent_mean = data['Base_percent'].mean()
self.base_percent_std = data['Base_percent'].std()
data['Base_percent'] = standardisation(data['Base_percent'],self.base_percent_mean,self.base_percent_std)
 self.age_mean = data['Age'].mean()
self.age_std = data['Age'].std()
data['Age'] = standardisation(data['Age'],self.age_mean,self.age_std)
 self.weeks_mean = data['Weeks'].mean()
self.weeks_std = data['Weeks'].std()
data['Weeks'] = standardisation(data['Weeks'],self.weeks_mean,self.weeks_std)
 if self.normalization:
self.base_week_min = data['Base_week'].min()
self.base_week_max = data['Base_week'].max()
data['Base_week'] = normalization(data['Base_week'],self.base_week_max,self.base_week_min)
 self.base_fvc_min = data['Base_FVC'].min()
self.base_fvc_max = data['Base_FVC'].max()
data['Base_FVC'] = normalization(data['Base_FVC'],self.base_fvc_max,self.base_fvc_min)
 self.base_percent_min = data['Percent'].min()
self.base_percent_max = data['Percent'].max()
data['Percent'] = normalization(data['Percent'],self.base_percent_max,self.base_percent_min)
 self.age_min = data['Age'].min()
self.age_max = data['Age'].max()
data['Age'] = normalization(data['Age'],self.age_max,self.age_min)
 self.weeks_min = data['Weeks'].min()
self.weeks_max = data['Weeks'].max()
data['Weeks'] = normalization(data['Weeks'],self.weeks_max,self.weeks_min)
 self.base_week_min = data['Min_week'].min()
self.base_week_max = data['Min_week'].max()
data['Min_week'] = normalization(data['Min_week'],self.base_week_max,self.base_week_min)
 return data
X_train, y_train, train_columns, _ = BuildFeature(is_train=True)
 X_test, _, _ , test_idx = BuildFeature(is_train=False )<drop_column>
pickefile = open('/kaggle/input/data-preparation-for-osic/data_prep', 'rb')
 data_prep = pickle.load(pickefile)
 pickefile.close()
X_train =reduce_mem_usage(X_train)
 X_test = reduce_mem_usage(X_test )<train_model>
X_prediction = data_prep(X_prediction)
LR_model = LinearRegression(n_jobs=4, normalize=True)
 LR_model.fit(X_train,y_train )<compute_test_metric>
LR_model.score(X_train,y_train )<predict_on_test>
pickefile = open('/kaggle/input/trained-cnn-mlp-for-osic/list_patient_score', 'rb')
 list_patient_score = pickle.load(pickefile)
 pickefile.close()
y_pred_train = LR_model.predict(X_train)
 y_pred_test = LR_model.predict(X_test )<feature_engineering>
list_patient_weight = []
for i in list_patient_score:
if 6.4>np.mean(i[1]):
list_patient_weight.append([i[0], 3])
 elif 6.8>np.mean(i[1])>6.4:
list_patient_weight.append([i[0], 50])
 elif 7.1>np.mean(i[1])>6.8:
list_patient_weight.append([i[0], 100])
 elif 50>np.mean(i[1])>7.6:
list_patient_weight.append([i[0], 3])
 else:
list_patient_weight.append([i[0], 3])
 train['Weight'] = train.Patient.map(dict(list_patient_weight))
y_pred_test[y_pred_test>1] = 1
y_pred_test[y_pred_test<0] = 0<save_to_csv>
pickefile = open('list_patient_weight', 'wb')
 pickle.dump(list_patient_weight, pickefile)
 pickefile.close()
df_test['winPlacePerc'] = y_pred_test
submission = df_test[['Id', 'winPlacePerc']]
submission.to_csv('submission.csv', index=False )<set_options>
list_patient_KFOLD=[]
for i in list_patient_score:
if np.mean(i[1])< 6.26:
list_patient_KFOLD.append([i[0], 0])
 elif 6.26<=np.mean(i[1])< 6.43:
list_patient_KFOLD.append([i[0], 1])
 elif 6.43<=np.mean(i[1])< 6.74:
list_patient_KFOLD.append([i[0], 2])
 elif 6.74<=np.mean(i[1])< 7.15:
list_patient_KFOLD.append([i[0], 3])
 elif 7.15<=np.mean(i[1])< 50:
list_patient_KFOLD.append([i[0], 4] )
sns.set_style('darkgrid')
 sns.set_palette('bone')
 pd.options.display.float_format = '{:,.3f}'.format
def toTapleList(list1,list2):
return list(itertools.product(list1,list2))
def reduce_mem_usage(df):

 start_mem = df.memory_usage().sum() / 1024**2
for col in df.columns:
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB --> {:.2f} MB(Decreased by {:.1f}%)'.format(
 start_mem, end_mem, 100 *(start_mem - end_mem)/ start_mem))
return df
train = pd.read_csv('.. /input/pubg-finish-placement-prediction/train_V2.csv')
 train = reduce_mem_usage(train)
 test = pd.read_csv('.. /input/pubg-finish-placement-prediction/test_V2.csv')
 test = reduce_mem_usage(test)
 print(train.shape, test.shape)
 print()
train.info()
print()
null_cnt = train.isnull().sum().sort_values()
print('null count:', null_cnt[null_cnt>0])
 train.dropna(inplace=True)
 print()
print(train.describe(include=np.number ).drop('count' ).T)
 print()
for c in ['Id','groupId','matchId']:
print(f'unique [{c}] count:',train[c].nunique())
 print()
fig, ax=plt.subplots(1,2,figsize=(12,4))
train.groupby('matchId')['matchType'].first().value_counts().plot.bar(ax=ax[0])
 mapper = lambda x: 'solo' if('solo' in x)else 'duo' if('duo' in x)or('crash' in x)else 'squad'
train['matchType'] = train['matchType'].apply(mapper)
 train.groupby('matchId')['matchType'].first().value_counts().plot.bar(ax=ax[1])
 plt.pause(15)
 print()
for q in ['numGroups == maxPlace', 'numGroups != maxPlace']:
print(q, ':', len(train.query(q)))
 print()
cols = ['numGroups','maxPlace']
desc1 = train.groupby('matchType')[cols].describe() [toTapleList(cols,['min','mean','max'])]
group = train.groupby(['matchType', 'matchId', 'groupId'] ).count().groupby(['matchType','matchId'] ).size().to_frame('groups_in_match')
 desc2 = group.groupby('matchType' ).describe() [toTapleList(['groups_in_match'],['min','mean','max'])]
group_res = pd.concat([desc1,desc2],axis=1)
 print(group_res)
 print()
match = train.groupby(['matchType','matchId'] ).size().to_frame('players_in_match')
 group = train.groupby(['matchType','matchId','groupId'] ).size().to_frame('players_in_group')
 player_res = pd.concat([match.groupby('matchType' ).describe() [toTapleList(['players_in_match'],['min','mean','max'])],
group.groupby('matchType' ).describe() [toTapleList(['players_in_group'],['min','mean','max'])]],axis=1)
 print(player_res)
 print()
print(group['players_in_group'].nlargest(5))
del match,group
print()
subset = train[train['matchId']=='41a634f62f86b7']
sub_group = subset[subset['groupId'] == '128b07271aa012']
print('matchId=='41a634f62f86b7' & groupId=='128b07271aa012'')
 print('-'*50)
 print('players:',len(subset))
print('groups:',subset['groupId'].nunique())
 print('maxPlace:',subset['maxPlace'].nunique())
 print('-'*50)
 print('max-group players:', len(sub_group))
print('max-group winPlacePerc:', sub_group['winPlacePerc'].unique())
 print('-'*50)
 print('winPlacePerc:', subset['winPlacePerc'].sort_values().unique())
 print()
group = train.groupby(['matchId','groupId','matchType'])['Id'].count().to_frame('players' ).reset_index()
group.loc[group['players']>4, 'players'] = 'default'
group['players'] = group['players'].astype(str)
 fig, ax = plt.subplots(1,3,figsize=(16,4))
for mt,ax in zip(['solo','duo','squad'], ax.ravel()):
ax.set_xlabel(mt)
 group[group['matchType'] == mt]['players'].value_counts().sort_index().plot.bar(ax=ax)
 plt.pause(15)
 print()
fig, ax = plt.subplots(1,2,figsize=(12,4))
train['matchDuration'].hist(bins=50,ax=ax[0])
 train.query('matchDuration >= 1400 & matchDuration <= 1800')['matchDuration'].hist(bins=50,ax=ax[1])
 plt.pause(15)
 print()
print(train[train['matchDuration'] == train['matchDuration'].min() ].head())
 print()
print(train[train['matchDuration'] == train['matchDuration'].max() ].head())
 print()
duartion_res =(train.groupby('matchId')['matchDuration'].nunique() >1 ).any()
print(duartion_res)
 print()
fig, ax = plt.subplots(2,2,figsize=(16,8))
cols = ['boosts','heals']
for cols, ax in zip(cols, ax):
sub = train[['winPlacePerc', cols]].copy()
mv =(sub[cols].max() // 5)+1
sub[cols] = pd.cut(sub[cols], [5*x for x in range(0, mv)], right=False)
 sub.groupby(cols ).mean() ['winPlacePerc'].plot.bar(ax=ax[0])
 train[cols].hist(bins=20, ax=ax[1])
 plt.pause(15)
 print()
print('solo player no revives:', 'solo' in train.query('revives > 0')['matchType'].unique())
 print()
fig, ax = plt.subplots(1,2,figsize=(16,4))
col = 'revives'
sub = train.loc[~train['matchType'].str.contains('solo'),['winPlacePerc',col]].copy()
sub[col] = pd.cut(sub[col], [5*x for x in range(0,8)], right=False)
 sub.groupby(col ).mean() ['winPlacePerc'].plot.bar(ax=ax[0])
 train[col].hist(bins=20,ax=ax[1])
 plt.pause(15)
 print()
print(train.groupby(['matchType'])['killPlace'].describe() [['min','mean','max']])
 print()
plt.figure(figsize=(8,4))
col = 'killPlace'
sub = train[['winPlacePerc', col]].copy()
sub[col] = pd.cut(sub[col], [10*x for x in range(0,11)],right=False)
 sub.groupby(col ).mean() ['winPlacePerc'].plot.bar()
plt.pause(15)
 print()
subMatch = train[train['matchId'] == train['matchId'].min() ].sort_values(['winPlacePerc','killPlace'])
 cols = ['groupId', 'kills', 'winPlacePerc', 'killPlace']
killPlace_res = subMatch[cols]
print(killPlace_res)
 print()
fig, ax = plt.subplots(1,2, figsize=(16,4))
col = 'kills'
sub = train[['winPlacePerc',col]].copy()
sub[col] = pd.cut(sub[col], [5*x for x in range(0,20)],right=False)
 sub.groupby(col ).mean() ['winPlacePerc'].plot.bar(ax=ax[0])
 train[train['kills'] <20][col].hist(bins=20, ax=ax[1])
 plt.pause(15)
 print()
sub = train['matchType'].str.contains('solo')
 kill_summary_res = pd.concat([train.loc[sub].groupby('matchId')['kills'].sum().describe() , train.loc[~sub].groupby('matchId')['kills'].sum().describe() ],
keys=['solo', 'group'], axis=1 ).T
print(kill_summary_res)
 print()
fig, ax = plt.subplots(2, 2, figsize=(16,8))
cols = ['killStreaks', 'DBNOs']
for col, ax in zip(cols, ax):
sub = train[['winPlacePerc', col]].copy()
sub[col]=pd.cut(sub[col], 6)
 sub.groupby(col ).mean() ['winPlacePerc'].plot.bar(ax=ax[0])
 train[col].hist(bins=20, ax=ax[1])
 plt.pause(15)
 print()
fig,ax = plt.subplots(3,2,figsize=(16,12))
cols = ['headshotKills', 'roadKills', 'teamKills']
for col, ax in zip(cols, ax):
sub = train[['winPlacePerc',col]].copy()
sub.loc[sub[col] >= 5, col] = '5+'
sub[col] = sub[col].astype(str)
 sub.groupby(col ).mean() ['winPlacePerc'].plot.bar(ax=ax[0])
 train[col].hist(bins=20, ax=ax[1])
 plt.pause(15)
 print()
fig, ax = plt.subplots(1,2, figsize=(16,4))
col='assists'
sub = train[['winPlacePerc', col]].copy()
sub.loc[sub[col] >= 5, col] = '5+'
sub[col] = sub[col].astype(str)
 sub.groupby(col ).mean() ['winPlacePerc'].plot.bar(ax=ax[0])
 train[col].hist(bins=20, ax=ax[1])
 plt.pause(15)
 print()
assists_res = pd.concat([train[train['matchType'] == 'solo'].describe() ['assists'], train[train['matchType'] != 'solo'].describe() ['assists']], keys=['solo', 'group'], axis=1 ).T
print(assists_res)
 print()
fig, ax = plt.subplots(1,2, figsize=(16, 4))
col = 'longestKill'
sub = train[['winPlacePerc', col]].copy()
sub[col] = pd.cut(sub[col], 6)
 sub.groupby(col ).mean() ['winPlacePerc'].plot.bar(ax=ax[0])
 train[col].hist(bins=20, ax=ax[1])
 plt.pause(15)
 print()
fig, ax = plt.subplots(1,2, figsize=(16,4))
col = 'damageDealt'
sub = train[['winPlacePerc', col]].copy()
sub[col] = pd.cut(sub[col], 6)
 sub.groupby(col ).mean() ['winPlacePerc'].plot.bar(ax=ax[0])
 train[col].hist(bins=20, ax=ax[1])
 plt.pause(15)
 print()
damageDealt_res = train.query('damageDealt == 0 &(kills >0 | DBNOs >0)')[['damageDealt', 'kills', 'DBNOs', 'headshotKills', 'roadKills', 'teamKills']].head(20)
 print()
fig, ax = plt.subplots(3,2, figsize=(16,12))
cols = ['walkDistance', 'rideDistance', 'swimDistance']
for col, ax in zip(cols, ax):
sub = train[['winPlacePerc', col]].copy()
sub[col] = pd.cut(sub[col], 6)
 sub.groupby(col ).mean() ['winPlacePerc'].plot.bar(ax=ax[0])
 train[col].hist(bins=20, ax=ax[1])
 plt.pause(15)
 print()
sub = train[['walkDistance', 'rideDistance', 'swimDistance', 'winPlacePerc']].copy()
walk = train['walkDistance']
sub['walkDistanceBin'] = pd.cut(walk, [0, 0.001, walk.quantile (.25), walk.quantile (.5), walk.quantile (.75), 99999])
 sub['rideDistanceBin'] =(train['rideDistance'] > 0 ).astype(int)
 sub['swimDistanceBin'] =(train['swimDistance'] > 0 ).astype(int)
 fig,ax = plt.subplots(1, 3, figsize=(16,3), sharey=True)
 sub.groupby('walkDistanceBin' ).mean() ['winPlacePerc'].plot.bar(ax=ax[0])
 sub.groupby('rideDistanceBin' ).mean() ['winPlacePerc'].plot.bar(ax=ax[1])
 sub.groupby('swimDistanceBin' ).mean() ['winPlacePerc'].plot.bar(ax=ax[2])
 del sub, walk
plt.pause(15)
 print()
sub = train.query('walkDistance == 0 & kills == 0 & weaponsAcquired == 0 & 'solo' in matchType')
 print('count:', len(sub), 'winPlacePerc:', round(sub['winPlacePerc'].mean() ,3))
print()
kills_res = train.query('kills >3 &(headshotKills/kills)>= 0.8')
 print('kills >3 &(headshotKills/kills)>= 0.8')
 print('count: ', len(kills_res))
print('winPlacePerc: ', round(kills_res['winPlacePerc'].mean() ,3))
print()
fig, ax = plt.subplots(1,3, figsize=(16,4))
cols = ['killPoints', 'rankPoints', 'winPoints']
for col, ax in zip(cols, ax.ravel()):
train.plot.scatter(x=col, y='winPlacePerc', ax=ax)
 plt.pause(15)
 print()
winPlacePerc_summary = train['winPlacePerc'].describe()
print(winPlacePerc_summary)
 print()
print('unique match count: ', train['matchId'].nunique())
 maxPlacePerc = train.groupby('matchId')['winPlacePerc'].max()
print('match [not contain 1st place]: ', len(maxPlacePerc[maxPlacePerc != 1]))
del maxPlacePerc
sub = train[(train['maxPlace'] > 1)&(train['numGroups'] == 1)]
print('match [maxPlace >1 & numGroup == 1] : ', len(sub.groupby('matchId')))
 print(' - unique winPlacePerc: ', sub['winPlacePerc'].unique())
 print()
winPlace_res = pd.concat([train[train['winPlacePerc'] == 1].head(10), train[train['winPlacePerc'] == 0].head(10)], keys=['winPlacePerc_1', 'winPlacePerc+0'])
 print(winPlace_res)
 print()
cols = ['kills', 'teamKills', 'DBNOs', 'revives', 'assists', 'boosts', 'heals', 'damageDealt', 'walkDistance', 'rideDistance', 'swimDistance', 'weaponsAcquired']
aggs = ['count', 'min', 'mean', 'max']
grp = train.loc[train['matchType'].str.contains('solo')].groupby('matchId')
 grpSolo = grp[cols].sum()
grp = train.loc[~train['matchType'].str.contains('solo')].groupby('matchId')
 grpTeam = grp[cols].sum()
winPlacePerc_res = pd.concat([grpSolo.describe().T[aggs], grpTeam.describe().T[aggs]],keys=['solo','team'],axis=1)
 print(winPlacePerc_res)
 print()
print(grpSolo.nlargest(5,'kills'))
print()
print(grpTeam.nlargest(5,'kills'))
del grpSolo, grpTeam
print()
cols = ['kills', 'teamKills', 'DBNOs', 'revives', 'assists', 'boosts', 'heals', 'damageDealt', 'walkDistance', 'rideDistance', 'swimDistance', 'weaponsAcquired']
cols.extend(['killPlace', 'winPlacePerc'])
 group = train.groupby(['matchId', 'groupId'])[cols]
fig, ax = plt.subplots(3, 1, figsize=(12,18), sharey=True)
 for df, ax in zip([group.mean() , group.min() , group.max() ], ax.ravel()):
sns.heatmap(df.corr() , annot=True, linewidths=.6, fmt='.2f', vmax=1, vmin=-1, center=0, cmap='Blues', ax=ax)
 plt.pause(150)
 print()
def printMatchStats(matchIds):
for mid in matchIds:
subMatch = train[train['matchId'] == mid]
print('matchType:', subMatch['matchType'].values[0])
 grp1st = subMatch[subMatch['winPlacePerc'] == 1]
grpOther = subMatch[subMatch['winPlacePerc'] != 1]
print('players'.ljust(10), ' total:{:>3} 1st:{:>3} other:{:>3}'.format(len(subMatch), len(grp1st), len(grpOther)))
 for c in ['kills', 'teamKills', 'roadKills', 'DBNOs', 'revives', 'assists']:
print(c.ljust(10), ' total:{:>3} 1st:{:>3} other:{:>3}'.format(subMatch[c].sum() , grp1st[c].sum() , grpOther[c].sum()))
print('-'*50)
 print()
match = train.groupby(['matchId'])['Id'].count()
fullplayer = match[match==100].reset_index()
sampleMid = fullplayer['matchId'][0:5]
printMatchStats(sampleMid)
 print()
all_data = train.append(test, sort=False ).reset_index(drop=True)
 del train, test
print(gc.collect())
 print()
match = all_data.groupby('matchId')
 all_data['killsPerc'] = match['kills'].rank(pct=True ).values
all_data['killPlacePerc'] = match['killPlace'].rank(pct=True ).values
all_data['walkDistancePerc'] = match['walkDistance'].rank(pct=True ).values
all_data['walkPerc_killPerc'] = all_data['walkDistancePerc']/all_data['killsPerc']
print()
all_data['_totalDistance'] = all_data['rideDistance'] + all_data['walkDistance'] + all_data['swimDistance']
print()
def fillInfo(df, val):
numcols = df.select_dtypes(include='number' ).columns
cols = numcols[numcols != 'winPlacePerc']
df[df == np.Inf] = np.NaN
df[df == np.NINF] = np.NaN
for c in cols:
df[c].fillna(val,inplace=True)
 print()
all_data['_healthItems'] = all_data['heals'] + all_data['boosts']
all_data['_headshotKillRate'] = all_data['headshotKills']/all_data['kills']
all_data['_killPlaceOverMaxPlace'] = all_data['killPlace'] / all_data['maxPlace']
all_data['_killsOverWalkDistance'] = all_data['kills'] / all_data['walkDistance']
all_data['_killsOverDistance'] = all_data['kills'] / all_data['_totalDistance']
all_data['_walkDistancePerSec'] = all_data['walkDistance'] / all_data['matchDuration']
fillInfo(all_data,0)
 print()
all_data.drop(['boosts', 'heals', 'killStreaks', 'DBNOs'], axis=1, inplace=True)
 all_data.drop(['headshotKills','roadKills', 'vehicleDestroys'], axis=1, inplace=True)
 all_data.drop(['rideDistance','swimDistance', 'matchDuration'], axis=1, inplace=True)
 all_data.drop(['rankPoints','killPoints', 'winPoints'], axis=1, inplace=True)
 print()
match = all_data.groupby(['matchId'])
 group = all_data.groupby(['matchId', 'groupId', 'matchType'])
 agg_col = list(all_data.columns)
 exclude_agg_col = ['Id','matchId','groupId','matchType','maxPlace','numGroups','winPlacePerc']
for c in exclude_agg_col:
agg_col.remove(c)
 print(agg_col)
 sum_col = ['kills','killPlace','damageDealt','walkDistance','_healthItems']
print()
match_data = pd.concat([match.size().to_frame('m.players'),
match[sum_col].sum().rename(columns=lambda s: 'm.sum.' + s),
match[sum_col].max().rename(columns=lambda s: 'm.max.' + s),
match[sum_col].mean().rename(columns=lambda s: 'm.mean.' + s)], axis=1 ).reset_index()
match_data = pd.merge(match_data,group[sum_col].sum().rename(columns=lambda s: 'sum.' + s ).reset_index())
 match_data = reduce_mem_usage(match_data)
 print(match_data.shape)
 print()
minKills= all_data.sort_values(['matchId','groupId','kills','killPlace'] ).groupby(['matchId','groupId','kills'] ).first().reset_index().copy()
for n in np.arange(4):
c = 'kills_' + str(n)+'_Place'
nKills =(minKills['kills'] == n)
 minKills.loc[nKills, c] = minKills[nKills].groupby(['matchId'])['killPlace'].rank().values
match_data = pd.merge(match_data, minKills[nKills][['matchId','groupId',c]],how='left')
 match_data = reduce_mem_usage(match_data)
 del minKills,nKills
print()
print(match_data.shape)
 print()
print(match_data.head())
 print()
all_data = pd.concat([group.size().to_frame('players'),
group.mean() ,
group[agg_col].max().rename(columns=lambda s: 'max.' +s),
group[agg_col].min().rename(columns=lambda s: 'min.' +s)], axis=1 ).reset_index()
all_data = reduce_mem_usage(all_data)
 print(all_data)
 print()
numcols = all_data.select_dtypes(include='number' ).columns.values
numcols = numcols[numcols != 'winPlacePerc']
all_data = pd.merge(all_data,match_data)
 del match_data
gc.collect()
all_data['enemy.players'] = all_data['m.players'] - all_data['players']
for c in sum_col:
all_data['enemy.players'+c] =(all_data['m.sum.'+c]- all_data['sum.'+c])/all_data['enemy.players']
all_data['p.sum_msum.'+c] = all_data['m.sum.'+c] / all_data['m.sum.'+c]
all_data['p.max_mmean.'+c] = all_data['max.'+c] / all_data['m.mean.'+c]
all_data['p.max_msum.'+c] = all_data['max.'+c] / all_data['m.sum.'+c]
all_data['p.max_mmax.'+c] = all_data['max.'+c] / all_data['m.max.'+c]
all_data.drop(['m.sum.'+c, 'm.max.'+c], axis=1, inplace=True)
 fillInfo(all_data,0)
 print(all_data.shape)
 print()
match = all_data.groupby('matchId')
 matchRank = match[numcols].rank(pct=True ).rename(columns=lambda s:'rank.'+s)
 all_data = reduce_mem_usage(pd.concat([all_data,matchRank],axis=1))
rank_col = matchRank.columns
del matchRank
gc.collect()
match = all_data.groupby('matchId')
 matchRank = match[rank_col].max().rename(columns=lambda s:'max.'+s ).reset_index()
all_data = pd.merge(all_data,matchRank)
 for c in numcols:
all_data['rank.'+c] = all_data['rank.'+c] / all_data['max.rank.'+c]
all_data.drop(['max.rank.'+c], axis=1,inplace=True)
 del matchRank
gc.collect()
print(all_data.shape)
 print()
all_data = pd.concat([all_data,pd.get_dummies(all_data['matchType'])], axis=1)
 all_data.drop(['matchType'], axis=1, inplace=True)
 all_data['matchId'] = all_data['matchId'].apply(lambda x: int(x,16))
all_data['groupId'] = all_data['groupId'].apply(lambda x: int(x,16))
null_count = all_data.isnull().sum().sort_values()
print(null_count[null_count>0])
 cols = [col for col in all_data.columns if col not in ['Id','matchId','groupId']]
for i,t in all_data.loc[:,cols].dtypes.iteritems() :
if t == object:
all_data[i] = pd.factorize(all_data[i])[0]
all_data = reduce_mem_usage(all_data)
 all_data.head()
print()
X_train = all_data[all_data['winPlacePerc'].notnull() ].reset_index(drop=True)
 X_test = all_data[all_data['winPlacePerc'].isnull() ].drop(['winPlacePerc'], axis=1 ).reset_index(drop=True)
 del all_data
gc.collect()
Y_train = X_train.pop('winPlacePerc')
 X_test_group = X_test[['matchId','groupId']].copy()
train_matchId = X_train['matchId']
X_train.drop(['matchId','groupId'], axis=1, inplace=True)
 X_test.drop(['matchId','groupId'], axis=1, inplace=True)
 print(X_train.shape, X_test.shape)
 print(pd.DataFrame([[val for val in dir() ], [sys.getsizeof(eval(val)) for val in dir() ]],
index=['name','size'] ).T.sort_values('size', ascending=False ).reset_index(drop=True)[:10])
 print()
params = {'learning_rate': 0.05, 'objective': 'mae', 'metric':'mae', 'num_leaves': 128,
'verbose': 1, 'verbose': 1, 'bagging_fraction': 0.7, 'feature_fraction': 0.7
}
reg = lgb.LGBMRegressor(**params, n_estimators=10000)
 reg.fit(X_train, Y_train)
 pred = reg.predict(X_test, num_iteration=reg.best_iteration_)
 feature_importance = reg.feature_importances_
feature_importance = 100.0 *(feature_importance / feature_importance.max())
 sorted_idx = np.argsort(feature_importance)
 sorted_idx = sorted_idx[len(feature_importance)- 30:]
pos = np.arange(sorted_idx.shape[0])+.5
plt.figure(figsize=(12,8))
plt.barh(pos, feature_importance[sorted_idx], align='center')
 plt.yticks(pos, X_train.columns[sorted_idx])
 plt.xlabel('Relative Importance')
 plt.title('Variable Importance')
 plt.show()
print()
print(X_train.columns[np.argsort(-feature_importance)].values)
 X_test_group['_nofit.winPlacePerc'] = pred
group = X_test_group.groupby(['matchId'])
 X_test_group['winPlacePerc'] = pred
X_test_group['_rank.winPlacePerc'] = group['winPlacePerc'].rank(method='min')
 X_test = pd.concat([X_test, X_test_group], axis=1)
 fullgroup =(X_test['numGroups'] == X_test['maxPlace'])
 subset = X_test.loc[fullgroup]
X_test.loc[fullgroup, 'winPlacePerc'] =(subset['_rank.winPlacePerc'].values - 1)/(subset['maxPlace'].values - 1)
 subset = X_test.loc[~fullgroup]
gap = 1.0 /(subset['maxPlace'].values - 1)
 new_perc = np.around(subset['winPlacePerc'].values / gap)* gap
X_test.loc[~fullgroup, 'winPlacePerc'] = new_perc
X_test['winPlacePerc'] = X_test['winPlacePerc'].clip(lower=0,upper=1)
 X_test.loc[X_test['maxPlace'] == 0, 'winPlacePerc'] = 0
X_test.loc[X_test['maxPlace'] == 1, 'winPlacePerc'] = 1
X_test.loc[(X_test['maxPlace'] > 1)&(X_test['numGroups'] == 1), 'winPlacePerc'] = 0
print(X_test['winPlacePerc'].describe())
 test = pd.read_csv('.. /input/pubg-finish-placement-prediction/test_V2.csv')
 test['matchId'] = test['matchId'].apply(lambda x: int(x,16))
test['groupId'] = test['groupId'].apply(lambda x: int(x,16))
submission = pd.merge(test, X_test[['matchId','groupId','winPlacePerc']])
 submission = submission[['Id','winPlacePerc']]
submission.to_csv("submission.csv", index=False)
<load_from_csv>
train = pd.read_csv(".. /input/pubg-finish-placement-prediction/train_V2.csv")
 test = pd.read_csv(".. /input/pubg-finish-placement-prediction/test_V2.csv")
 gender_submission = pd.read_csv(".. /input/pubg-finish-placement-prediction/sample_submission_V2.csv" )<concatenate>
data = pd.concat([train, test], sort = True)
 print(len(train), len(test), len(data)) 
<drop_column>
KFOLD_confidence = [0.05, 0.15, 0.2, 0.25, 0.35]
delete_columns = ['Id', 'groupId', 'matchId']
data.drop(delete_columns, axis = 1, inplace = True )<feature_engineering>
pe = np.zeros(( X_prediction.shape[0], 3))
pred = np.zeros(( train.shape[0], 3))
for i in range(NFOLD):
print(f"FOLD {i}")
 model = create_model(LAMBDA_LOSS)
 list_patients = [j[0] for j in list_patient_KFOLD if j[1]==i]
history = model.fit(x=train[~train.Patient.isin(list_patients)][SELECTED_COLUMNS], y=train[~train.Patient.isin(list_patients)][['FVC']], validation_data=(train[train.Patient.isin(list_patients)][SELECTED_COLUMNS], train[train.Patient.isin(list_patients)][['FVC']]), epochs=EPOCH[i], sample_weight=train[~train.Patient.isin(list_patients)].Weight, verbose=0)
 print("train", model.evaluate(train[~train.Patient.isin(list_patients)][SELECTED_COLUMNS], train[~train.Patient.isin(list_patients)][['FVC']], verbose=0, batch_size=BATCH_SIZE))
print("val", model.evaluate(train[train.Patient.isin(list_patients)][SELECTED_COLUMNS], train[train.Patient.isin(list_patients)][['FVC']], verbose=0, batch_size=BATCH_SIZE))
pred[train[train.Patient.isin(list_patients)].index] = model.predict(train[train.Patient.isin(list_patients)][SELECTED_COLUMNS], batch_size=BATCH_SIZE, verbose=0)
 pe += model.predict(X_prediction[SELECTED_COLUMNS], batch_size=BATCH_SIZE, verbose=0)* KFOLD_confidence[i]
model.save('model_' + str(i))
data['fightRange'] =(data['kills'] + data['DBNOs'])/ data['walkDistance'] * 100
data['fightRange'].replace([np.inf,-np.inf], np.nan, inplace = True)
 data['fightRange'].fillna(0, inplace = True)
 data['surpriseAttackRatio'] =(data['kills'] + data['DBNOs'])/ data['damageDealt'] * 100
data['surpriseAttackRatio'].replace([np.inf,-np.inf], np.nan, inplace = True)
 data['surpriseAttackRatio'].fillna(0, inplace = True )<categorify>
defaultmatch = ['squad-fpp', 'duo', 'solo-fpp', 'squad', 'duo-fpp', 'solo',
'normal-squad-fpp', 'normal-solo-fpp','normal-duo-fpp',
'normal-duo', 'normal-squad', 'normal-solo']
crashmatch = ['crashfpp', 'crashtpp']
flarematch = ['flarefpp', 'flaretpp']
data['defaultMatch'] = data['matchType'].replace(defaultmatch, 1)
 data['defaultMatch'].replace(crashmatch, 0, inplace=True)
 data['defaultMatch'].replace(flarematch, 0, inplace=True)
 data['crashMatch'] = data['matchType'].replace(crashmatch, 1)
 data['crashMatch'].replace(defaultmatch, 0, inplace=True)
 data['crashMatch'].replace(flarematch, 0, inplace=True)
 data['flareMatch'] = data['matchType'].replace(flarematch, 1)
 data['flareMatch'].replace(defaultmatch, 0, inplace=True)
 data['flareMatch'].replace(crashmatch, 0, inplace=True )<drop_column>
sigma_opt = mean_absolute_error(train[['FVC']], pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean)
 X_prediction['FVC1'] = 0.996*pe[:, 1]
X_prediction['Confidence1'] = pe[:, 2] - pe[:, 0]
data.drop('matchType', axis = 1, inplace = True )<prepare_x_and_y>
subm = X_prediction.copy()
subm['FVC'] = 3020
subm['Confidence'] = 100
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
train = data[:len(train)]
test = data[len(train):]
train = train.dropna()
y_test_pred = 0<drop_column>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )
<import_modules><EOS>