Kaggle competition: ASHRAE - Great Energy Predictor III
<SOS> metric: RMSLE Kaggle data source: ashrae-great-energy-predictor-iii<init_hyperparams>
py.init_notebook_mode(connected=True)
<init_hyperparams>
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df
params = {'iterations': 1000, 'depth': 6,
'learning_rate': 0.037587603778015836, 'random_strength': 0.6401109404028098,
'bagging_temperature': 0.10845008746433943, 'l2_leaf_reg': 6.247622793136706,
'has_time': True}<create_dataframe>
%%time
root = Path('.. /input/ashrae-feather-format-for-fast-loading')
 train_df = pd.read_feather(root/'train.feather')
 test_df = pd.read_feather(root/'test.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather' )
train_pool = Pool(df_X, df_y0, cat_features=categorical_features_indices )<train_model>
leak_df = pd.read_feather('.. /input/ashrae-leak-data-station/leak.feather')
 leak_df.fillna(0, inplace=True)
 leak_df = leak_df[(leak_df.timestamp.dt.year > 2016)&(leak_df.timestamp.dt.year < 2019)]
leak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0
leak_df = leak_df[leak_df.building_id!=245]
model = CatBoostClassifier(**params, 
)
 model.fit(train_pool,
verbose=True,
)<create_dataframe>
leak_df.meter.value_counts()
importance = pd.DataFrame(model.feature_importances_, index=df_X.columns, columns=['importance'])
 importance.sort_values("importance", ascending=False ).head(50 )<import_modules>
print(leak_df.duplicated().sum() )
from kaggle.competitions import nflrush<set_options>
print(len(leak_df)/ len(train_df))
warnings.simplefilter('ignore' )<feature_engineering>
del train_df
gc.collect()
for(test_df, sample_prediction_df)in tqdm(env.iter_test()):
df_a = preprocess(test_df)
 df_a = preprocess_coord(df_a)
 df_ = df_a.loc[df_a["NflId"]==df_a["NflIdRusher"], :]
df_ = get_team_score(df_)
 df_a = player_team_name(df_a)
 df_ = team_name(df_)
 df_a.loc[:, "offense"] =(df_a["PlayerTeamAbbr"]==df_a["PossessionTeam"])*1
df_a["position"] = df_a["Position"].map(position_rename)
 df_["position"] = df_["Position"].map(position_rename)
 df_a.loc[df_a["offense"]==1, "position"] = df_a.loc[df_a["offense"]==1, "position"].apply(group_position_offense)
 df_a.loc[df_a["offense"]==0, "position"] = df_a.loc[df_a["offense"]==0, "position"].apply(group_position_defense)
 df_ = df_.merge(position_stats(df_a), on="PlayId", how="left")
 df_year = df_
test_X=pd.DataFrame(df_year,columns=df_X.columns)
 test_X.fillna(means)
 pred520 = model.predict_proba(test_X)[0]
pred100 = np.concatenate([ np.zeros(85), pred520[0]*small_probs["value"].values, pred520[1:-1], pred520[-1]*large_probs["value"].values ])
 sample_prediction_df.iloc[0] = pred100.cumsum()
sample_prediction_df.loc[:, "Yards"+str(test_X["yardsToTouchdown"][0]): ] = 1
sample_prediction_df[sample_prediction_df>1.0] = 1.0
env.predict(sample_prediction_df)
<save_to_csv>
sample_submission1 = pd.read_csv('.. /input/ashrae-kfold-lightgbm-without-leak-1-08/submission.csv', index_col=0)
 sample_submission2 = pd.read_csv('.. /input/ashrae-half-and-half/submission.csv', index_col=0)
 sample_submission3 = pd.read_csv('.. /input/ashrae-highway-kernel-route4/submission.csv', index_col=0 )
env.write_submission_file()<define_variables>
test_df['pred1'] = sample_submission1.meter_reading
test_df['pred2'] = sample_submission2.meter_reading
test_df['pred3'] = sample_submission3.meter_reading
test_df.loc[test_df.pred3<0, 'pred3'] = 0
del sample_submission1, sample_submission2, sample_submission3
gc.collect()
test_df = reduce_mem_usage(test_df)
 leak_df = reduce_mem_usage(leak_df )
( test_df, sample_prediction_df)= next(env.iter_test() )<feature_engineering>
leak_df = leak_df.merge(test_df[['building_id', 'meter', 'timestamp', 'pred1', 'pred2', 'pred3', 'row_id']], left_on = ['building_id', 'meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'], how = "left")
 leak_df = leak_df.merge(building_meta_df[['building_id', 'site_id']], on='building_id', how='left' )
df_ = df_a.loc[df_a["NflId"]==df_a["NflIdRusher"], :]
df_ = get_team_score(df_)
 df_a = player_team_name(df_a)
 df_ = team_name(df_)
 df_a.loc[:, "offense"] =(df_a["PlayerTeamAbbr"]==df_a["PossessionTeam"])*1
df_a["position"] = df_a["Position"].map(position_rename)
 df_["position"] = df_["Position"].map(position_rename)
 df_a.loc[df_a["offense"]==1, "position"] = df_a.loc[df_a["offense"]==1, "position"].apply(group_position_offense)
 df_a.loc[df_a["offense"]==0, "position"] = df_a.loc[df_a["offense"]==0, "position"].apply(group_position_defense)
 df_ = df_.merge(position_stats(df_a), on="PlayId", how="left")
 df_year = df_play
test_X=pd.DataFrame(df_year,columns=df_X.columns)
 test_X.fillna(means)
 pred520 = model.predict_proba(test_X)[0]
pred100 = np.concatenate([ np.zeros(85), pred520[0]*small_probs["value"].values, pred520[1:-1], pred520[-1]*large_probs["value"].values ])
 sample_prediction_df.iloc[0] = pred100.cumsum()
sample_prediction_df.loc[:, "Yards"+str(test_X["yardsToTouchdown"][0]): ] = 1
sample_prediction_df[sample_prediction_df>1.0] = 1.0<filter>
leak_df['pred1_l1p'] = np.log1p(leak_df.pred1)
 leak_df['pred2_l1p'] = np.log1p(leak_df.pred2)
 leak_df['pred3_l1p'] = np.log1p(leak_df.pred3)
 leak_df['meter_reading_l1p'] = np.log1p(leak_df.meter_reading )
sample_prediction_df.loc[:, "Yards-17":]<categorify>
leak_df[leak_df.pred1_l1p.isnull() ]
df_all = preprocess(test_df )<filter>
leak_df['mean_pred'] = np.mean(leak_df[['pred1', 'pred2', 'pred3']].values, axis=1)
 leak_df['mean_pred_l1p'] = np.log1p(leak_df.mean_pred)
 leak_score = np.sqrt(mean_squared_error(leak_df.mean_pred_l1p, leak_df.meter_reading_l1p))
sns.distplot(leak_df.mean_pred_l1p)
 sns.distplot(leak_df.meter_reading_l1p)
 print('mean score=', leak_score )
df_ = df_all[df_all["NflId"]==df_all["NflIdRusher"]]<feature_engineering>
N = 10
scores = np.zeros(N,)
 for i in range(N):
p = i * 1./N
v = p * leak_df['pred1'].values +(1.-p)* leak_df ['pred3'].values
vl1p = np.log1p(v)
 scores[i] = np.sqrt(mean_squared_error(vl1p, leak_df.meter_reading_l1p))
<compute_train_metric>
best_weight = np.argmin(scores)* 1./N
print(scores.min() , best_weight )
df_ = get_team_score(df_)
 df_ = average_personal_data(df_, df_all)
 df_ = add_last_yards(df_ )<merge>
scores = np.zeros(N,)
 for i in range(N):
p = i * 1./N
v = p *(best_weight * leak_df['pred1'].values +(1.-best_weight)* leak_df ['pred3'].values)+(1.-p)* leak_df ['pred2'].values
vl1p = np.log1p(v)
 scores[i] = np.sqrt(mean_squared_error(vl1p, leak_df.meter_reading_l1p))
df_year = merge_lastYearAv(df_, team_yards_df, defence_yards_df, personal_yards, 2019 )<concatenate>
best_weight2 = np.argmin(scores)* 1./N
print(scores.min() , best_weight2)
test_X=pd.concat([train_df,test_X],sort=False)
 test_X = test_X.fillna(0 )<predict_on_test>
X_train = np.array([leak_df['pred1'].values,leak_df['pred2'].values, leak_df['pred3'].values] ).T
y_train = leak_df.meter_reading_l1p
pred_df = pd.DataFrame(clf.predict_proba(test_X), columns=[ "Yards"+str(i)for i in clf.classes_])
 pred_df = pred_df.cumsum(axis=1 )<set_options>
weights = tf.Variable([[0.3],[0.3],[0.3]])
 steps = 1000
lr = 0.1
opt = tf.optimizers.SGD(lr)
 @tf.function()
def train_step(opt):
with tf.GradientTape() as tape:
y = tf.matmul(X_train, weights)[:,0]
loss = tf.reduce_mean(( tf.math.log1p(y)- y_train)** 2)
 grads = tape.gradient(loss, weights)
 opt.apply_gradients([(grads, weights)])
 return loss
prev_loss = 9999
for i in range(steps):
loss = train_step(opt)
 if loss > prev_loss:
lr /= 2
opt.lr = lr
prev_loss = loss
print(f'step: {i} {loss.numpy() }')
warnings.filterwarnings('ignore')
 tf.disable_v2_behavior()
__print__ = print
def print(string):
os.system(f'echo "{string}"')
 __print__(string )<load_from_csv>
np.sqrt(loss )
df = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2020/train.csv', low_memory=False)
 shape = str(df.shape)
 print(df.columns)
 print(df.shape )<feature_engineering>
sample_submission = pd.read_feather(os.path.join(root, 'sample_submission.feather'))
ws = weights.numpy()
w1 = ws[0,0]
w2 = ws[1,0]
w3 = ws[2,0]
print("The weights are: w1=" + str(w1)+ ", w2=" + str(w2)+ ", w3=" + str(w3))
sample_submission['meter_reading'] = w1 * test_df.pred1 + w2 * test_df.pred2 + w3 * test_df.pred3
sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0
def prepare_data(df):
df.loc[df['StadiumType'].isnull() , 'StadiumType'] = 'Unknown'
df.loc[df['GameWeather'].isnull() , 'GameWeather'] = 'Unknown'
df.loc[df['WindDirection'].isnull() , 'WindDirection'] = 'Unknown'
df.loc[df['WindSpeed'].isnull() , 'WindSpeed'] = '0'
df.loc[df['Orientation'].isnull() , 'Orientation'] = 0
df.loc[df['Dir'].isnull() , 'Dir'] = 0
df.loc[df['GameWeather']== 'T: 51; H: 55; W: NW 10 mph', 'Temperature'] = 51
df.loc[df['GameWeather']== 'T: 51; H: 55; W: NW 10 mph', 'WindDirection'] = 'NW 10 mph'
df.loc[df['GameWeather']== 'T: 51; H: 55; W: NW 10 mph', 'Humidity'] = 55
df.loc[df['Temperature'].isnull() , 'Temperature'] = 70
df.loc[df['Humidity'].isnull() , 'Humidity'] = 0
df.loc[df['OffenseFormation'].isnull() , 'OffenseFormation'] = 'Unknown'
df.loc[df['PossessionTeam'] == df['HomeTeamAbbr'] , 'DefendingTeam'] = df['VisitorTeamAbbr']
df.loc[df['PossessionTeam'] != df['HomeTeamAbbr'] , 'DefendingTeam'] = df['HomeTeamAbbr']
df.loc[(df['FieldPosition'].isnull())&(df['PlayDirection'] == 'right')&(df['X'] >= 50), 'FieldPosition'] = df['DefendingTeam']
df.loc[(df['FieldPosition'].isnull())&(df['PlayDirection'] == 'right')&(df['X'] < 50), 'FieldPosition'] = df['PossessionTeam']
df.loc[(df['FieldPosition'].isnull())&(df['PlayDirection'] == 'left')&(df['X'] >= 50), 'FieldPosition'] = df['PossessionTeam']
df.loc[(df['FieldPosition'].isnull())&(df['PlayDirection'] == 'left')&(df['X'] < 50), 'FieldPosition'] = df['DefendingTeam']
df = df.dropna()
df['WindSpeed'] = df['WindSpeed'].astype('str')
 df['WindSpeed'] = df['WindSpeed'].str.replace('mph', '')
 df['WindSpeed'] = df['WindSpeed'].str.replace('Mph', '')
 df['WindSpeed'] = df['WindSpeed'].str.replace('MPH', '')
 df['WindSpeed'] = df['WindSpeed'].str.replace('MPh', '')
 df['WindSpeed'] = df['WindSpeed'].str.strip()
b = df['WindSpeed'].loc[df['WindSpeed'].str.contains('-', regex=False)==True].unique()
b1 = df['WindSpeed'].loc[df['WindSpeed'].str.contains('-', regex=False)==True].str[:2]
b2 = df['WindSpeed'].loc[df['WindSpeed'].str.contains('-', regex=False)==True].str[-2:]
b3 =(b1.astype('float64')+ b2.astype('float64')) / 2
for i in b:
df.loc[(df.WindSpeed == i), 'WindSpeed'] = b3
v = df[df['WindSpeed'].str.isnumeric() == False]
av = v['WindSpeed'].unique()
for j in av:
df.loc[(df.WindSpeed == j), 'WindSpeed'] = '0.0'
df['WindSpeed'] = df['WindSpeed'].astype('float64')
 df.X[df['PlayDirection'] == 'left'] = 120 - df['X']
df.YardLine[df['PossessionTeam'] == df['FieldPosition']] = 10 + df['YardLine']
df.YardLine[df['PossessionTeam'] != df['FieldPosition']] = 60.0 +(50 - df['YardLine'])
 df.Orientation[df['PlayDirection'] == 'left'] = 360 - df['Orientation']
df.loc[(df['PlayDirection'] == 'left')&(df['Orientation'] == 0),'Orientation'] = 0
df.Orientation[df['PlayDirection'] == 'left'] = 360 - df['Dir']
df.loc[(df['PlayDirection'] == 'left')&(df['Dir'] == 0),'Dir'] = 0
nfl = {
1:	'ARI',
2:	'ATL',
3:	'BAL',
4:	'BUF',
5:	'CAR',
6:	'CHI',
7:	'CIN',
8:	'CLE',
9:	'DAL',
10:	'DEN',
11:	'DET',
12:	'GB',
13:	'HOU',
14:	'IND',
15:	'JAX',
16:	'KC',
17:	'LAC',
18:	'LAR',
19:	'MIA',
20:	'MIN',
21:	'NE',
22:	'NO',
23:	'NYG',
24:	'NYJ',
25:	'OAK',
26:	'PHI',
27:	'PIT',
28:	'SEA',
29:	'SF',
30:	'TB',
31:	'TEN',
32:	'WAS'
}
nfl_teams = pd.DataFrame.from_dict(nfl, orient='index')
 nfl_teams['team_id'] = nfl_teams.index
nfl_teams = nfl_teams.rename(columns={0: 'team_abb'})
 	
df.loc[df['HomeTeamAbbr'] == 'ARZ', 'HomeTeamAbbr'] = 'ARI'
df.loc[df['HomeTeamAbbr'] == 'BLT', 'HomeTeamAbbr'] = 'BAL'
df.loc[df['HomeTeamAbbr'] == 'CLV', 'HomeTeamAbbr'] = 'CLE'
df.loc[df['HomeTeamAbbr'] == 'HST', 'HomeTeamAbbr'] = 'HOU'
df.loc[df['HomeTeamAbbr'] == 'LA', 'HomeTeamAbbr'] = 'LAR'
df.loc[df['VisitorTeamAbbr'] == 'ARZ', 'VisitorTeamAbbr'] = 'ARI'
df.loc[df['VisitorTeamAbbr'] == 'BLT', 'VisitorTeamAbbr'] = 'BAL'
df.loc[df['VisitorTeamAbbr'] == 'CLV', 'VisitorTeamAbbr'] = 'CLE'
df.loc[df['VisitorTeamAbbr'] == 'HST', 'VisitorTeamAbbr'] = 'HOU'
df.loc[df['VisitorTeamAbbr'] == 'LA', 'VisitorTeamAbbr'] = 'LAR'
df = df.merge(nfl_teams, left_on ='HomeTeamAbbr', right_on ='team_abb', how = 'left')
 df = df.drop(['HomeTeamAbbr','team_abb'], axis=1)
 df = df.rename(columns={'team_id': 'HomeTeamAbbr'})
 df = df.merge(nfl_teams, left_on ='VisitorTeamAbbr', right_on ='team_abb', how = 'left')
 df = df.drop(['VisitorTeamAbbr','team_abb'], axis=1)
 df = df.rename(columns={'team_id': 'VisitorTeamAbbr'})
 df['HomeTeamAbbr'] = df['HomeTeamAbbr'].astype('float64')
 df['VisitorTeamAbbr'] = df['VisitorTeamAbbr'].astype('float64')
 	
df.loc[df['FieldPosition'] == 'ARZ', 'FieldPosition'] = 'ARI'
df.loc[df['FieldPosition'] == 'BLT', 'FieldPosition'] = 'BAL'
df.loc[df['FieldPosition'] == 'CLV', 'FieldPosition'] = 'CLE'
df.loc[df['FieldPosition'] == 'HST', 'FieldPosition'] = 'HOU'
df.loc[df['FieldPosition'] == 'LA', 'FieldPosition'] = 'LAR'
df.loc[df['PossessionTeam'] == 'ARZ', 'PossessionTeam'] = 'ARI'
df.loc[df['PossessionTeam'] == 'BLT', 'PossessionTeam'] = 'BAL'
df.loc[df['PossessionTeam'] == 'CLV', 'PossessionTeam'] = 'CLE'
df.loc[df['PossessionTeam'] == 'HST', 'PossessionTeam'] = 'HOU'
df.loc[df['PossessionTeam'] == 'LA', 'PossessionTeam'] = 'LAR'
df = df.merge(nfl_teams, left_on ='FieldPosition', right_on ='team_abb', how = 'left')
 df = df.drop(['FieldPosition','team_abb'], axis=1)
 df = df.rename(columns={'team_id': 'FieldPosition'})
 df = df.merge(nfl_teams, left_on ='PossessionTeam', right_on ='team_abb', how = 'left')
 df = df.drop(['PossessionTeam','team_abb'], axis=1)
 df = df.rename(columns={'team_id': 'PossessionTeam'})
 df['FieldPosition'] = df['FieldPosition'].astype('float64')
 df['PossessionTeam'] = df['PossessionTeam'].astype('float64')
 	
df['DefensePersonnel_DL'] = df['DefensePersonnel'].str.replace(', ', '_' ).str.replace(' ', '' ).str.slice(0, 1)
 df['DefensePersonnel_LB'] = df['DefensePersonnel'].str.replace(', ', '_' ).str.replace(' ', '' ).str.slice(4, 5)
 df['DefensePersonnel_DB'] = df['DefensePersonnel'].str.replace(', ', '_' ).str.replace(' ', '' ).str.slice(8, 9)
 df['DefensePersonnel_DL'] = df['DefensePersonnel_DL'].astype('int64')
 df['DefensePersonnel_LB'] = df['DefensePersonnel_LB'].astype('int64')
 df['DefensePersonnel_DB'] = df['DefensePersonnel_DB'].astype('int64')
 	
df.loc[df['OffensePersonnel'].str.replace(', ', '_' ).str.replace(' ', '' ).str.slice(1, 3)== 'OL', 'OffensePersonnel_Flag'] = 1
df.loc[df['OffensePersonnel'].str.replace(', ', '_' ).str.replace(' ', '' ).str.slice(1, 3)== 'QB', 'OffensePersonnel_Flag'] = 2
df.loc[df['OffensePersonnel'].str.replace(', ', '_' ).str.replace(' ', '' ).str.slice(1, 3)== 'RB', 'OffensePersonnel_Flag'] = 0
	
df.loc[df['OffensePersonnel_Flag'] == 2, 'OffensePersonnel_QB'] = df['OffensePersonnel'].str.replace(', ', '_' ).str.replace(' ', '' ).str.slice(0, 1)
 df.loc[df['OffensePersonnel_Flag'] == 2, 'OffensePersonnel_RB'] = df['OffensePersonnel'].str.replace(', ', '_' ).str.replace(' ', '' ).str.slice(4, 5)
 df.loc[df['OffensePersonnel_Flag'] == 2, 'OffensePersonnel_TE'] = df['OffensePersonnel'].str.replace(', ', '_' ).str.replace(' ', '' ).str.slice(8, 9)
 df.loc[df['OffensePersonnel_Flag'] == 2, 'OffensePersonnel_WR'] = df['OffensePersonnel'].str.replace(', ', '_' ).str.replace(' ', '' ).str.slice(12, 13)
 df.loc[df['OffensePersonnel_Flag'] != 2, 'OffensePersonnel_QB'] = '1'
	
df.loc[df['OffensePersonnel_Flag'] == 1, 'OffensePersonnel_OL'] = df['OffensePersonnel'].str.replace(', ', '_' ).str.replace(' ', '' ).str.slice(0, 1)
 df.loc[df['OffensePersonnel_Flag'] == 1, 'OffensePersonnel_RB'] = df['OffensePersonnel'].str.replace(', ', '_' ).str.replace(' ', '' ).str.slice(4, 5)
 df.loc[df['OffensePersonnel_Flag'] == 1, 'OffensePersonnel_TE'] = df['OffensePersonnel'].str.replace(', ', '_' ).str.replace(' ', '' ).str.slice(8, 9)
 df.loc[df['OffensePersonnel_Flag'] == 1, 'OffensePersonnel_WR'] = df['OffensePersonnel'].str.replace(', ', '_' ).str.replace(' ', '' ).str.slice(12, 13)
 	
df.loc[df['OffensePersonnel_Flag'] == 0, 'OffensePersonnel_RB'] = df['OffensePersonnel'].str.replace(', ', '_' ).str.replace(' ', '' ).str.slice(0, 1)
 df.loc[df['OffensePersonnel_Flag'] == 0, 'OffensePersonnel_TE'] = df['OffensePersonnel'].str.replace(', ', '_' ).str.replace(' ', '' ).str.slice(4, 5)
 df.loc[df['OffensePersonnel_Flag'] == 0, 'OffensePersonnel_WR'] = df['OffensePersonnel'].str.replace(', ', '_' ).str.replace(' ', '' ).str.slice(8, 9)
 	
df['OffensePersonnel_QB'] = df['OffensePersonnel_QB'].astype('int64')
 df['OffensePersonnel_RB'] = df['OffensePersonnel_RB'].astype('int64')
 df['OffensePersonnel_TE'] = df['OffensePersonnel_TE'].astype('int64')
 df['OffensePersonnel_WR'] = df['OffensePersonnel_WR'].astype('int64')
 df.loc[df['OffensePersonnel_Flag'] != 1, 'OffensePersonnel_OL_tmp'] = 11 -(df['OffensePersonnel_RB'] + df['OffensePersonnel_TE'] + df['OffensePersonnel_WR']+df['OffensePersonnel_QB'])
 df.loc[df['OffensePersonnel_Flag'] != 1, 'OffensePersonnel_OL'] = df['OffensePersonnel_OL_tmp']
df['OffensePersonnel_OL'] = df['OffensePersonnel_OL'].astype('int64')
 df = df.drop(['OffensePersonnel_OL_tmp'], axis=1)
 	
	
	
PlayerHeight_ft=df['PlayerHeight'].str.split('-' ).str[0].astype('float64')
 PlayerHeight_in=df['PlayerHeight'].str.split('-' ).str[1].astype('float64')
 PlayerHeight_m = PlayerHeight_ft*0.3048 + PlayerHeight_in*0.0254
df['PlayerHeight'] = PlayerHeight_m.astype('float64')
 	
time_snap_handoff = pd.to_datetime(df['TimeHandoff'])- pd.to_datetime(df['TimeSnap'])
 time_snap_handoff_sec = pd.to_timedelta(time_snap_handoff)
 df['TimeSnapHandoff'] = time_snap_handoff_sec.dt.seconds.astype('int64')
 df['GameYear'] = pd.DatetimeIndex(df['TimeSnap'] ).year
df['GameMonth'] = pd.DatetimeIndex(df['TimeSnap'] ).month
df['GameDay'] = pd.DatetimeIndex(df['TimeSnap'] ).day
today=dt.datetime.today().strftime('%Y-%m-%d')
 today=pd.to_datetime(today)
 PlayerAgeDays = today - pd.to_datetime(df['PlayerBirthDate'])
 df['PlayerAgeYears'] = PlayerAgeDays/np.timedelta64(1,'Y')
 GameClockMin = pd.DatetimeIndex(df['GameClock'] ).hour
GameClockSec = pd.DatetimeIndex(df['GameClock'] ).minute
df['GameClockSeconds'] = GameClockMin*60 + GameClockSec
columns_to_drop = ['GameClock','TimeSnap','TimeHandoff','PlayerBirthDate', 'JerseyNumber',
'OffensePersonnel', 'DefensePersonnel','PlayerCollegeName']
df = df.drop(columns_to_drop, axis=1)
 df.columns
return df<categorify>
leak_df = leak_df[['meter_reading', 'row_id']].set_index('row_id' ).dropna()
sample_submission.loc[leak_df.index, 'meter_reading'] = leak_df['meter_reading']
<categorify><EOS> 
sample_submission.to_csv('submission.csv', index=False, float_format='%.4f' )