Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<save_to_csv>
plt.style.use('ggplot')
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu" )
test_id = test_data["Id"]
submit_xgb = pd.DataFrame({'Id': test_id, "winPlacePerc": preds} , columns=['Id', 'winPlacePerc'])
 print(submit_xgb.head())
 submit_xgb.to_csv("submission.csv", index = False )<set_options>
def seed_all(seed=42):
np.random.seed(seed)
 torch.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_all()
warnings.filterwarnings("ignore")
 gc.enable()
print(os.listdir(".. /input")) 
<load_from_csv>
def calc_metric_loss(pred_fvc,sigma,true_fvc):
true_fvc=np.reshape(true_fvc,pred_fvc.shape)
 sigma[sigma<70]=70
delta=np.abs(pred_fvc-true_fvc)
 delta[delta>1000]=1000
metric=-(np.sqrt(2)*delta/sigma)-np.log(np.sqrt(2)*sigma)
 return -metric
def feature_engineering(is_train=True,debug=True):
if is_train:
print("processing train.csv")
 if debug == True:
df = pd.read_csv('.. /input/train_V2.csv', nrows=10000)
 else:
df = pd.read_csv('.. /input/train_V2.csv')
 df = df[df['maxPlace'] > 1]
else:
print("processing test.csv")
 df = pd.read_csv('.. /input/test_V2.csv')
 print("remove some columns")
 target = 'winPlacePerc'
print("Adding Features")
 df['headshotrate'] = df['kills']/df['headshotKills']
df['killStreakrate'] = df['killStreaks']/df['kills']
df['healthitems'] = df['heals'] + df['boosts']
df['totalDistance'] = df['rideDistance'] + df["walkDistance"] + df["swimDistance"]
df['killPlace_over_maxPlace'] = df['killPlace'] / df['maxPlace']
df['headshotKills_over_kills'] = df['headshotKills'] / df['kills']
df['distance_over_weapons'] = df['totalDistance'] / df['weaponsAcquired']
df['walkDistance_over_heals'] = df['walkDistance'] / df['heals']
df['walkDistance_over_kills'] = df['walkDistance'] / df['kills']
df['killsPerWalkDistance'] = df['kills'] / df['walkDistance']
df["skill"] = df["headshotKills"] + df["roadKills"]
df[df == np.Inf] = np.NaN
df[df == np.NINF] = np.NaN
print("Removing Na's From DF")
 df.fillna(0, inplace=True)
 features = list(df.columns)
 features.remove("Id")
 features.remove("matchId")
 features.remove("groupId")
 features.remove("matchType")
 y = None
if is_train:
print("get target")
 y = df.groupby(['matchId','groupId'])[target].agg('mean')
 features.remove(target)
 print("get group mean feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('mean')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
if is_train: df_out = agg.reset_index() [['matchId','groupId']]
else: df_out = df[['matchId','groupId']]
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 print("get group max feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('max')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId', 'groupId'])
 print("get group min feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('min')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId', 'groupId'])
 print("get group size feature")
 agg = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_out = df_out.merge(agg, how='left', on=['matchId', 'groupId'])
 print("get match mean feature")
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
df_out = df_out.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 print("get match size feature")
 agg = df.groupby(['matchId'] ).size().reset_index(name='match_size')
 df_out = df_out.merge(agg, how='left', on=['matchId'])
 df_out.drop(["matchId", "groupId"], axis=1, inplace=True)
 X = df_out
del df, df_out, agg, agg_rank
gc.collect()
return X, y<prepare_x_and_y>
class main_model(nn.Module):
def __init__(self, n_additional_features, n_outputs):
super(main_model, self ).__init__()
self.fc1 = nn.Linear(n_additional_features, 100)
 self.fc2 = nn.Linear(100, 100)
 self.fc3 = nn.Linear(100, n_outputs)
 def forward(self, additional_features):
out=additional_features
out = F.relu(self.fc1(out))
out = F.relu(self.fc2(out))
out = self.fc3(out)
 return out
def metric_loss(self,pred_fvc,true_fvc):
sigma = pred_fvc[:, 2] - pred_fvc[:, 0]
true_fvc=torch.reshape(true_fvc,pred_fvc[:,1].shape)
 sigma_clipped=torch.clamp(sigma,min=70)
 delta=torch.clamp(torch.abs(pred_fvc[:,1]-true_fvc),max=1000)
 metric=torch.div(-torch.sqrt(torch.tensor([2.0] ).to(device)) *delta,sigma_clipped)-torch.log(torch.sqrt(torch.tensor([2.0] ).to(device)) *sigma_clipped)
 return -metric
def fvc_loss(self,pred_fvc,true_fvc):
true_fvc=torch.reshape(true_fvc,pred_fvc[:,1].shape)
 fvc_err=torch.abs(pred_fvc-true_fvc)
 return fvc_err
def quantile_loss(self,preds, target, quantiles):
assert not target.requires_grad
assert preds.size(0)== target.size(0)
 losses = []
for i, q in enumerate(quantiles):
errors = target - preds[:, i]
losses.append(torch.max(( q - 1)* errors, q * errors ).unsqueeze(1))
loss = torch.mean(torch.sum(torch.cat(losses, dim=1), dim=1))
return loss
x_train, y_train = feature_engineering(True,False)
 x_test, _ = feature_engineering(False,True )<drop_column>
def load_scan(path):
slices = [pydicom.read_file(path / s)for s in os.listdir(path)]
slices.sort(key = lambda x: float(x.ImagePositionPatient[2]))
try:
slice_thickness = np.abs(slices[0].ImagePositionPatient[2] - slices[1].ImagePositionPatient[2])
 except:
slice_thickness = np.abs(slices[0].SliceLocation - slices[1].SliceLocation)
 if slice_thickness==0:
slice_thickness=slices[0].SliceThickness
for s in slices:
s.SliceThickness = slice_thickness
return slices
def get_pixels_hu(slices):
image = np.stack([np.array(s.pixel_array,dtype=np.int16)for s in slices])
 image = image.astype(np.int16)
 image[image == -2000] = 0
for slice_number in range(len(slices)) :
intercept = slices[slice_number].RescaleIntercept
slope = slices[slice_number].RescaleSlope
if slope != 1:
image[slice_number] = slope * image[slice_number].astype(np.float64)
 image[slice_number] = image[slice_number].astype(np.int16)
 image[slice_number] += np.int16(intercept)
 return np.array(image, dtype=np.int16)
 def resample(image, scan, new_spacing=[1,1,1]):
spacing = np.array([scan[0].SliceThickness] + list(scan[0].PixelSpacing), dtype=np.float32)
 resize_factor = spacing / new_spacing
new_real_shape = image.shape * resize_factor
new_shape = np.round(new_real_shape)
 real_resize_factor = new_shape / image.shape
new_spacing = spacing / real_resize_factor
image = scipy.ndimage.interpolation.zoom(image, real_resize_factor, mode='nearest')
 return image, new_spacing
def get_segmented_lungs(im, plot=False):

 if plot == True:
f, plots = plt.subplots(8, 1, figsize=(5, 40))

 binary = im < -200
if plot == True:
plots[0].axis('off')
 plots[0].imshow(binary, cmap=plt.cm.bone)

cleared = clear_border(binary)
 if plot == True:
plots[1].axis('off')
 plots[1].imshow(cleared, cmap=plt.cm.bone)

label_image = label(cleared)
 if plot == True:
plots[2].axis('off')
 plots[2].imshow(label_image, cmap=plt.cm.bone)

areas = [r.area for r in regionprops(label_image)]
areas.sort()
if len(areas)> 2:
for region in regionprops(label_image):
if region.area < areas[-2]:
for coordinates in region.coords:
label_image[coordinates[0], coordinates[1]] = 0
binary = label_image > 0
if plot == True:
plots[3].axis('off')
 plots[3].imshow(binary, cmap=plt.cm.bone)

selem = disk(2)
 binary = binary_erosion(binary, selem)
 if plot == True:
plots[4].axis('off')
 plots[4].imshow(binary, cmap=plt.cm.bone)

selem = disk(10)
 binary = binary_closing(binary, selem)
 if plot == True:
plots[5].axis('off')
 plots[5].imshow(binary, cmap=plt.cm.bone)

edges = roberts(binary)
 binary = ndi.binary_fill_holes(edges)
 if plot == True:
plots[6].axis('off')
 plots[6].imshow(binary, cmap=plt.cm.bone)

get_high_vals = binary == 0
im[get_high_vals] = 0
if plot == True:
plots[7].axis('off')
 plots[7].imshow(im, cmap=plt.cm.bone)
 return im
def get_kurtosis_stats(ids,ctscans_dir):
kurt=[]
std=[]
mean=[]
median=[]
for i in ids:
print(i)
 patient_path= ctscans_dir / i
scan = load_scan(patient_path)
 image=get_pixels_hu(scan)
 image, new_spacing = resample(image, scan, new_spacing=[2,2,2])
 image=np.asarray([get_segmented_lungs(slice)for slice in image])
 kurt_i=kurtosis(image.ravel() [image.ravel() < -200])
 std_i=image.ravel() [image.ravel() < -200].std()
mean_i=image.ravel() [image.ravel() < -200].mean()
median_i=np.median(image.ravel() [image.ravel() < -200])
 print('Kurtosis: ', kurt_i)
 print('Standard Deviation: ', std_i)
 kurt.append(kurt_i)
 std.append(std_i)
 mean.append(mean_i)
 median.append(median_i)
 ax=sns.kdeplot(image.ravel() [(image.ravel() < 0)&(image.ravel() > -1200)], bw=0.5)
 ax.set(xlabel='HU', ylabel='% voxels',title='Histogram of voxel characteristics')
 plt.show()
plt.imshow(image[round(image.shape[0]/2),:,:])
 plt.show()
return kurt,std,mean,median
x_train = reduce_mem_usage(x_train)
 x_test = reduce_mem_usage(x_test )<load_from_csv>
train=pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv')
 test=pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 submission=pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv' )
def post_rst(pred_test):
df_sub = pd.read_csv(".. /input/sample_submission_V2.csv")
 df_test = pd.read_csv(".. /input/test_V2.csv")
 df_sub['winPlacePerc'] = pred_test
df_sub = df_sub.merge(df_test[["Id", "matchId", "groupId", "maxPlace", "numGroups"]], on="Id", how="left")
 df_sub_group = df_sub.groupby(["matchId", "groupId"] ).first().reset_index()
df_sub_group["rank"] = df_sub_group.groupby(["matchId"])["winPlacePerc"].rank()
df_sub_group = df_sub_group.merge(
 df_sub_group.groupby("matchId")["rank"].max().to_frame("max_rank" ).reset_index() ,
on="matchId", how="left")
 df_sub_group["adjusted_perc"] =(df_sub_group["rank"] - 1)/(df_sub_group["numGroups"] - 1)
 df_sub = df_sub.merge(df_sub_group[["adjusted_perc", "matchId", "groupId"]], on=["matchId", "groupId"], how="left")
 df_sub["winPlacePerc"] = df_sub["adjusted_perc"]
df_sub.loc[df_sub.maxPlace == 0, "winPlacePerc"] = 0
df_sub.loc[df_sub.maxPlace == 1, "winPlacePerc"] = 1
subset = df_sub.loc[df_sub.maxPlace > 1]
gap = 1.0 /(subset.maxPlace.values - 1)
 new_perc = np.around(subset.winPlacePerc.values / gap)* gap
df_sub.loc[df_sub.maxPlace > 1, "winPlacePerc"] = new_perc
df_sub.loc[(df_sub.maxPlace > 1)&(df_sub.numGroups == 1), "winPlacePerc"] = 0
assert df_sub["winPlacePerc"].isnull().sum() == 0
df_sub[["Id", "winPlacePerc"]].to_csv("submission.csv", index=False )<choose_model_class>
train['base_Weeks']=train.groupby(['Patient'])['Weeks'].transform('min')
 base=train[train.Weeks==train.base_Weeks]
base = base.rename(columns={'FVC': 'base_FVC','Percent': 'base_Percent'})
 base.drop_duplicates(subset=['Patient', 'Weeks'], keep='first',inplace=True)
 train=train.merge(base[['Patient','base_FVC','base_Percent']],on='Patient',how='left')
 train['Week_passed'] = train['Weeks'] - train['base_Weeks']
def nn_model(input_shape=x_train.shape[1], hidden_size=64):
model = Sequential()
model.add(Dense(hidden_size, input_dim=input_shape, kernel_initializer='normal'))
model.add(LeakyReLU(0.1))
model.add(Dense(hidden_size, kernel_initializer='normal'))
model.add(LeakyReLU(0.1))
model.add(Dense(hidden_size, kernel_initializer='normal'))
model.add(LeakyReLU(0.1))
model.add(Dense(hidden_size, kernel_initializer='normal'))
model.add(LeakyReLU(0.1))
model.add(Dense(1, kernel_initializer='normal'))
model.compile(loss='mean_absolute_error', optimizer='adam', metrics=['mae'])
 return model<split>
test = test.rename(columns={'Weeks': 'base_Weeks', 'FVC': 'base_FVC','Percent': 'base_Percent'})
 submission = pd.read_csv(".. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv")
 submission['Patient'] = submission['Patient_Week'].apply(lambda x:x.split('_')[0])
 submission['Weeks'] = submission['Patient_Week'].apply(lambda x:x.split('_')[1] ).astype(int)
 test = submission.drop(columns = ["FVC", "Confidence"] ).merge(test, on = 'Patient')
 test['Week_passed'] = test['Weeks'] - test['base_Weeks']
test=test[train.columns.drop(['FVC','Percent'])]
def train_val_split(x_train, y_train):
train_index = round(int(x_train.shape[0]*0.8))
dev_X = x_train[:train_index]
val_X = x_train[train_index:]
dev_y = y_train[:train_index]
val_y = y_train[train_index:]
del x_train, y_train
gc.collect() ;
return dev_X, val_X, dev_y, val_y<split>
COLS = ['Sex','SmokingStatus']
for col in COLS:
for mod in train[col].unique() :
train[mod] =(train[col] == mod ).astype(int)
 test[mod] =(test[col] == mod ).astype(int)
 train.drop(col,axis=1,inplace=True)
 test.drop(col,axis=1,inplace=True )
dev_X, val_X, dev_y, val_y = train_val_split(x_train, y_train)
 def run_lgb(train_X, train_y, val_X, val_y, x_test):
params = {"objective" : "regression", "metric" : "mae", 'n_estimators':5000, 'early_stopping_rounds':200,
"num_leaves" : 150, "learning_rate" : 0.05, "bagging_fraction" : 0.5,
"bagging_seed" : 0, "num_threads" : 4,"colsample_bytree" : 0.5
}
lgtrain = lgb.Dataset(train_X, label=train_y)
 lgval = lgb.Dataset(val_X, label=val_y)
 model = lgb.train(params, lgtrain, valid_sets=[lgtrain, lgval], early_stopping_rounds=200, verbose_eval=1000)
 pred_test_y = model.predict(x_test, num_iteration=model.best_iteration)
 return pred_test_y
pred_test_lgb = run_lgb(dev_X, dev_y, val_X, val_y, x_test)
 pred_test_lgb = pred_test_lgb.reshape(-1,1)
 del dev_X, val_X, dev_y, val_y
gc.collect()<train_model>
pixel_stats=pd.read_csv('.. /input/osic-histogram-features/train_pixel_stats.csv')
 train=train.merge(pixel_stats[['Patient','kurtosis','std','mean','median']],how='left',on='Patient' )
np.random.seed(42)
 std_scaler = StandardScaler().fit(x_train)
 x_train = std_scaler.transform(x_train)
 x_test = std_scaler.transform(x_test)
 mlp_model = nn_model(x_train.shape[1], hidden_size=64)
 mlp_model.fit(x_train, y_train, batch_size=128, epochs=8, verbose=1, validation_split=0.1, shuffle=True)
 pred_test_mlp = mlp_model.predict(x_test)
 del mlp_model
gc.collect()<drop_column>
robust_scaler = preprocessing.RobustScaler()
train[train.columns.difference(['Patient','FVC','Percent','Weeks','base_Weeks'])]=robust_scaler.fit_transform(train[train.columns.difference(['Patient','FVC','Percent','Weeks','base_Weeks'])] )
del x_train, y_train, x_test
gc.collect()<define_variables>
epochs=300
batch_size=64
num_workers=3
quantiles =(0.2, 0.5, 0.8 )
pred_test =(pred_test_mlp + pred_test_lgb)/ 2.0<set_options>
ids=train.Patient.unique()
index = np.argwhere(ids=='ID00011637202177653955184')
 ids = list(np.delete(ids, index))
random.shuffle(ids)
 ids=np.array(ids)
 train_ids,val_ids=np.split(ids, [int(round(0.9 * len(ids), 0)) ])
 train_dataset = OSIC(train_ids,train)
 train_dataloader = DataLoader(train_dataset, batch_size=batch_size,shuffle=True,num_workers=num_workers)
 val_dataset = OSIC(val_ids,train)
 val_dataloader = DataLoader(val_dataset, batch_size=batch_size,shuffle=True,num_workers=num_workers )
gc.enable()
dtypes = {
		'assists' : 'uint8',
		'boosts' : 'uint8',
		'damageDealt' : 'float16',
		'DBNOs' : 'uint8',
		'headshotKills' : 'uint8',
		'heals' : 'uint8',
		'killPlace' : 'uint8',
		'killPoints' : 'uint8',
		'kills' : 'uint8',
		'killStreaks' : 'uint8',
		'longestKill' : 'float16',
		'maxPlace' : 'uint8',
		'numGroups' : 'uint8',
		'revives' : 'uint8',
		'rideDistance' : 'float16',
		'roadKills' : 'uint8',
		'swimDistance' : 'float16',
		'teamKills' : 'uint8',
		'vehicleDestroys' : 'uint8',
		'walkDistance' : 'float16',
		'weaponsAcquired' : 'uint8',
		'winPoints' : 'uint8',
		'winPlacePerc' : 'float16'
}
def feature_engineering(filename,train = False):
	data = pd.read_csv(filename,dtype=dtypes)
 	data = data[data['maxPlace'] > 1]
	data['headshotrate'] = data['kills']/data['headshotKills']
	data['killStreakrate'] = data['killStreaks']/data['kills']
	data['healthitems'] = data['heals'] + data['boosts']
	data['totalDistance'] = data['rideDistance'] + data["walkDistance"] + data["swimDistance"]
	data['killPlace_over_maxPlace'] = data['killPlace'] / data['maxPlace']
	data['headshotKills_over_kills'] = data['headshotKills'] / data['kills']
	data['distance_over_weapons'] = data['totalDistance'] / data['weaponsAcquired']
	data['walkDistance_over_heals'] = data['walkDistance'] / data['heals']
	data['walkDistance_over_kills'] = data['walkDistance'] / data['kills']
	data['killsPerWalkDistance'] = data['kills'] / data['walkDistance']
	data["skill"] = data["headshotKills"] + data["roadKills"]
	data[data == np.Inf] = np.NaN
	data[data == np.NINF] = np.NaN
	data.fillna(0, inplace=True)
 	feature = list(data.columns)
 	feature.remove('Id')
 	feature.remove('groupId')
 	feature.remove('matchId')
 	feature.remove('matchType')
 	if(train):
		labels = np.array(data.groupby(['matchId','groupId'])['winPlacePerc'].agg('mean'), dtype=np.float64)
 		feature.remove('winPlacePerc')
 	else:
		labels = data[['Id']]
	
	print("group_max")
 	agg = data.groupby(['matchId','groupId'])[feature].agg('max')
 	agg_rank = agg.groupby('matchId')[feature].rank(pct=True ).reset_index()
	if train: data_out = agg.reset_index() [['matchId','groupId']]
	else: data_out = data[['matchId','groupId']]
	data_out = data_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId','groupId'])
 	data_out = data_out.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId','groupId'])
 	
	print("group_mean")
 	agg = data.groupby(['matchId','groupId'])[feature].agg('mean')
 	agg_rank = agg.groupby('matchId')[feature].rank(pct=True ).reset_index()
	data_out = data_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId','groupId'])
 	data_out = data_out.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId','groupId'])
 	
	print("group_min")
 	agg = data.groupby(['matchId','groupId'])[feature].agg('min')
 	agg_rank = agg.groupby('matchId')[feature].rank(pct=True ).reset_index()
	data_out = data_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId','groupId'])
 	data_out = data_out.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId','groupId'])
 	
	print("match_mean")
 	agg = data.groupby(['matchId'])[feature].agg('mean' ).reset_index()
	data_out = data_out.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 	
	print("match_max")
 	agg = data.groupby(['matchId'])[feature].agg('max' ).reset_index()
	data_out = data_out.merge(agg, suffixes=["", "_match_max"], how='left', on=['matchId'])
 	
	print("match_size")
 	agg = data.groupby(['matchId'] ).size().reset_index(name='match_size')
 	data_out = data_out.merge(agg, how='left', on=['matchId'])
 	
	del data,agg,agg_rank
	gc.collect()
	data_out.drop(["matchId", "groupId"], axis=1, inplace=True)
 	data_out = reduce_size(data_out)
 	X = data_out
	del data_out, feature
	gc.collect()
	return X,labels
def reduce_size(merged_data_out):
	print(' Starting size is %d Mb'%(sys.getsizeof(merged_data_out)/1024/1024))
	print(' Columns: %d'%(merged_data_out.shape[1]))
	feats = merged_data_out.columns[merged_data_out.dtypes == 'float64']
	for feat in feats:
		merged_data_out[feat] = merged_data_out[feat].astype('float32')
 	feats = merged_data_out.columns[merged_data_out.dtypes == 'int16']
	for feat in feats:
		mm = np.abs(merged_data_out[feat] ).max()
		if mm < 126:
			merged_data_out[feat] = merged_data_out[feat].astype('int8')
 	feats = merged_data_out.columns[merged_data_out.dtypes == 'int32']
	for feat in feats:
		mm = np.abs(merged_data_out[feat] ).max()
		if mm < 126:
			merged_data_out[feat] = merged_data_out[feat].astype('int8')
 		elif mm < 30000:
			merged_data_out[feat] = merged_data_out[feat].astype('int16')
 	feats = merged_data_out.columns[merged_data_out.dtypes == 'int64']
	for feat in feats:
		mm = np.abs(merged_data_out[feat] ).max()
		if mm < 126:
			merged_data_out[feat] = merged_data_out[feat].astype('int8')
 		elif mm < 30000:
			merged_data_out[feat] = merged_data_out[feat].astype('int16')
 		elif mm < 2000000000:
			merged_data_out[feat] = merged_data_out[feat].astype('int32')
 	print(' Ending size is %d Mb'%(sys.getsizeof(merged_data_out)/1024/1024))
	return merged_data_out
params = {
	'objective': 'regression',
	'early_stopping_rounds':200,
	'n_estimators':20000,
	'metric': 'mae',
	"bagging_seed" : 0,
	'num_leaves': 31,
	'learning_rate': 0.05,
	'bagging_fraction': 0.9,
	"num_threads" : 4,
	"colsample_bytree" : 0.7
}
if __name__ == '__main__':
	batch_size = 512
	num_of_features = 0
	
	
	trainpath = '.. /input/pubg-finish-placement-prediction/train_V2.csv'
	testpath = '.. /input/pubg-finish-placement-prediction/test_V2.csv'
	features, labels = feature_engineering(trainpath,train = True)
 	num_of_features = features.shape[1]
	
	filepath = "best.h5"
	split = int(len(labels)*0.8)
 	lgb_train = lgb.Dataset(features[:split], labels[:split])
 	lgb_val = lgb.Dataset(features[split:], labels[split:])
 	del features, labels
	gc.collect()
	gbm = lgb.train(params, lgb_train, verbose_eval=100,valid_sets=[lgb_train, lgb_val],early_stopping_rounds = 200)
 	del lgb_train,lgb_val
	gc.collect()
	features_test, test = feature_engineering(testpath)
 	predict = gbm.predict(features_test,num_iteration=gbm.best_iteration)
 	del features_test
	gc.collect()
	predict = predict.reshape(-1)
 	test['winPlacePerc'] = predict
	df_test = pd.read_csv(testpath)
 	
	test = test.merge(df_test[["Id", "matchId", "groupId", "maxPlace", "numGroups"]], on="Id", how="left")
 	
	df_sub_group = test.groupby(["matchId", "groupId"] ).first().reset_index()
	df_sub_group["rank"] = df_sub_group.groupby(["matchId"])["winPlacePerc"].rank()
	df_sub_group["adjusted_perc"] =(df_sub_group["rank"] - 1)/(df_sub_group["numGroups"] - 1)
 	test = test.merge(df_sub_group[["adjusted_perc", "matchId", "groupId"]], on=["matchId", "groupId"], how="left")
 	test["winPlacePerc"] = test["adjusted_perc"]
	
	test.loc[test.maxPlace == 0, "winPlacePerc"] = 0
	test.loc[test.maxPlace == 1, "winPlacePerc"] = 1
	
	
	subset = test.loc[test.maxPlace > 1]
	gap = 1.0 /(subset.maxPlace.values - 1)
 	new_perc = np.around(subset.winPlacePerc.values / gap)* gap
	test.loc[test.maxPlace > 1, "winPlacePerc"] = new_perc
	
	test.loc[(test.maxPlace > 1)&(test.numGroups == 1), "winPlacePerc"] = 0
	assert test["winPlacePerc"].isnull().sum() == 0
	test[["Id", "winPlacePerc"]].to_csv("submission.csv", index=False )<load_from_csv>
model = main_model(train_dataset.data.shape[1], 3 ).to(device)
 print('Number of parameters:')
 print(sum(p.numel() for p in model.parameters() if p.requires_grad))
act_train = pd.read_csv('.. /input/act_train.csv')
 act_test = pd.read_csv('.. /input/act_test.csv')
 people = pd.read_csv('.. /input/people.csv')
 people.sample(10 )<feature_engineering>
optimizer = optim.Adam(model.parameters())
 epoch_train_metric=[]
epoch_val_metric=[]
epoch_train_loss=[]
epoch_val_loss=[]
epoch=0
min_val_loss = 1e+100
early_stop = False
patience=10
while epoch<epochs and not early_stop:
epoch+=1
train_loss=0
train_metric=0
model.train()
for batch_idx, data in enumerate(train_dataloader):
optimizer.zero_grad()
model_output = model(data['data'].float().to(device))
quantile_loss = model.quantile_loss(model_output, data['fvc'].to(device), quantiles)
 metric_loss = model.metric_loss(model_output,data['fvc'].to(device)).mean()
loss=quantile_loss
loss.backward()
train_loss += loss.item()
train_metric += metric_loss.item()
optimizer.step()
print('====> Epoch: {} Average train quantile loss: {:.4f}'.format(
 epoch, train_loss / len(train_dataloader)))
 print('====> Epoch: {} Average train metric: {:.4f}'.format(
 epoch, train_metric / len(train_dataloader)))
 val_loss=0
val_metric=0
model.eval()
with torch.no_grad() :
for batch_idx, data in enumerate(val_dataloader):
model_output = model(data['data'].float().to(device))
quantile_loss = model.quantile_loss(model_output, data['fvc'].to(device), quantiles)
 metric_loss = model.metric_loss(model_output,data['fvc'].to(device)).mean()
loss=quantile_loss
val_loss += loss.item()
val_metric += metric_loss.item()
print('====> Epoch: {} Average val quantile loss: {:.4f}'.format(
 epoch, val_loss / len(val_dataloader)))
 print('====> Epoch: {} Average val metric: {:.4f}'.format(
 epoch, val_metric / len(val_dataloader)))
 epoch_train_loss.append(train_loss/ len(train_dataloader))
epoch_val_loss.append(val_loss / len(val_dataloader))
epoch_train_metric.append(train_metric/ len(train_dataloader))
epoch_val_metric.append(val_metric / len(val_dataloader))
if val_loss < min_val_loss:
min_val_loss = val_loss
best_model = copy.deepcopy(model.state_dict())
 print('Min loss %0.2f' % min_val_loss)
 epochs_no_improve = 0
else:
epochs_no_improve += 1
if epochs_no_improve == patience:
print('Early stopping!')
 early_stop = True
model.load_state_dict(best_model)
def process_dates(data,min_date):
min_date
data=data.apply(lambda x:(datetime.datetime.strptime(x,"%Y-%m-%d")
 -datetime.datetime.strptime(min_date,"%Y-%m-%d")).days)
 data
return data<data_type_conversions>
submission=pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv' )
test_ids = act_test['activity_id']
def preprocess_acts(data,min_date, train_set=True):
dates=data['date']
dates=process_dates(dates,min_date)
 data = data.drop(['date', 'activity_id'], axis=1)
 if(train_set):
data = data.drop(['outcome'], axis=1)
 data['people_id'] = data['people_id'].apply(lambda x: x.split('_')[1])
 data['people_id'] = pd.to_numeric(data['people_id'] ).astype(int)
 columns = list(data.columns)
 for col in columns[1:]:
data[col] = data[col].fillna('type 0')
 data[col] = data[col].apply(lambda x: x.split(' ')[1])
 data[col] = pd.to_numeric(data[col] ).astype(int)
 data['dates']=dates
return data
def preprocess_people(data,min_date):
dates=data['date']
dates=process_dates(dates,min_date)
 data = data.drop(['date'], axis=1)
 data['people_id'] = data['people_id'].apply(lambda x: x.split('_')[1])
 data['people_id'] = pd.to_numeric(data['people_id'] ).astype(int)
 columns = list(data.columns)
 bools = columns[11:]
strings = columns[1:11]
for col in bools:
data[col] = pd.to_numeric(data[col] ).astype(int)
 for col in strings:
data[col] = data[col].fillna('type 0')
 data[col] = data[col].apply(lambda x: x.split(' ')[1])
 data[col] = pd.to_numeric(data[col] ).astype(int)
 data['dates']=dates
return data<concatenate>
test_ids=test.Patient.unique()
test_dataset = OSIC(test_ids,test,train=False)
 test_dataloader = DataLoader(test_dataset, batch_size=batch_size,shuffle=False,num_workers=num_workers)
 model.eval()
fvc_pred = []
sigma_pred = []
with torch.no_grad() :
for batch_idx, data in enumerate(test_dataloader):
model_output = model(data['data'].float().to(device))
fvc_pred.append(model_output[:,1])
 sigma_pred.append(model_output[:,2]-model_output[:,0])
 fvc_pred=torch.cat(fvc_pred, dim=0)
 sigma_pred=torch.cat(sigma_pred, dim=0)
 test['FVC']=fvc_pred.cpu().numpy()
test['Confidence']=sigma_pred.cpu().numpy()
min_date=pd.concat([people['date'],act_train['date'],act_test['date']] ).min()
min_date<categorify>
test['Patient_Week']=test["Patient"] + '_' + test['Weeks'].apply(str )
min_date=pd.concat([people['date'],act_train['date'],act_test['date']] ).min()
peeps = preprocess_people(people,min_date)
 actions_train = preprocess_acts(act_train,min_date,train_set=True)
 actions_test = preprocess_acts(act_test,min_date,train_set=False)
 print(peeps.columns)
 print(actions_train.columns)
 peeps.sample(10 )<merge>
submission=submission[['Patient_Week']].merge(test[['Patient_Week','FVC','Confidence']],on='Patient_Week' )
features = actions_train.merge(peeps, how='left', on='people_id')
 features=features.drop(['people_id'],axis=1)
 labels = act_train['outcome']
test = actions_test.merge(peeps, how='left', on='people_id')
 test=test.drop(['people_id'],axis=1)
 features.sample(10 )<train_model>
submission.to_csv('submission.csv', index=False, float_format='%.1f' )
num_test = 0.10
X_train, X_test, y_train, y_test = train_test_split(features, labels, test_size=num_test, random_state=23)
 clf = RandomForestClassifier()
clf.fit(X_train, y_train )<compute_train_metric>
fvc_pred = []
sigma_pred = []
train_dataloader = DataLoader(train_dataset, batch_size=batch_size,shuffle=False,num_workers=num_workers)
 with torch.no_grad() :
for batch_idx, data in enumerate(train_dataloader):
model_output = model(data['data'].float().to(device))
fvc_pred.append(model_output[:,1])
 sigma_pred.append(model_output[:,2]-model_output[:,0])
 fvc_pred_train=torch.cat(fvc_pred, dim=0)
 sigma_pred_train=torch.cat(sigma_pred, dim=0)
 print('train metric', calc_metric_loss(fvc_pred_train.cpu().numpy() ,sigma_pred_train.cpu().numpy() ,train_dataset.fvc ).mean())
 plt.scatter(fvc_pred_train.cpu().numpy() ,sigma_pred_train.cpu().numpy())
 plt.title('Train')
 plt.xlabel('FVC')
 plt.ylabel('Confidence')
<prepare_output><EOS> 
fvc_pred = []
sigma_pred = []
val_dataloader = DataLoader(val_dataset, batch_size=batch_size,shuffle=False,num_workers=num_workers)
 with torch.no_grad() :
for batch_idx, data in enumerate(val_dataloader):
model_output = model(data['data'].float().to(device))
fvc_pred.append(model_output[:,1])
 sigma_pred.append(model_output[:,2]-model_output[:,0])
 fvc_pred_val=torch.cat(fvc_pred, dim=0)
 sigma_pred_val=torch.cat(sigma_pred, dim=0)
 print('val metric', calc_metric_loss(fvc_pred_val.cpu().numpy() ,sigma_pred_val.cpu().numpy() ,val_dataset.fvc ).mean())
 plt.scatter(fvc_pred_val.cpu().numpy() ,sigma_pred_val.cpu().numpy())
 plt.title('Val')
 plt.xlabel('FVC')
 plt.ylabel('Confidence' )