Kaggle competition: ASHRAE - Great Energy Predictor III
<SOS> metric: RMSLE Kaggle data source: ashrae-great-energy-predictor-iii<save_to_csv>
py.init_notebook_mode(connected=True)
X_train = X_gameid.join(X_features)
 print(X_train.shape)
 print(y_train.shape)
 X_train.head()
X_train_2017 = X_train.query("Season < 1.0")
 X_train_2018 = X_train.query("Season >= 1.0")
 X_train[feature_name].to_csv('train_feat.csv')
 X_train_2017.to_csv('train17.csv')
 X_train_2018.to_csv('train18.csv' )<train_model>
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df
class CRPSCallback(Callback):
def __init__(self,validation, predict_batch_size=20, include_on_batch=False):
super(CRPSCallback, self ).__init__()
self.validation = validation
self.predict_batch_size = predict_batch_size
self.include_on_batch = include_on_batch
print('validation shape',len(self.validation))
def on_batch_begin(self, batch, logs={}):
pass
def on_train_begin(self, logs={}):
if not('CRPS_score_val' in self.params['metrics']):
self.params['metrics'].append('CRPS_score_val')
 def on_batch_end(self, batch, logs={}):
if(self.include_on_batch):
logs['CRPS_score_val'] = float('-inf')
 def on_epoch_end(self, epoch, logs={}):
logs['CRPS_score_val'] = float('-inf')
 if(self.validation):
X_valid, y_valid = self.validation[0], self.validation[1]
y_pred = self.model.predict(X_valid)
 y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * X_valid.shape[0])
 val_s = np.round(val_s, 6)
 logs['CRPS_score_val'] = val_s<choose_model_class>
%%time
root = Path('.. /input/ashrae-feather-format-for-fast-loading')
 train_df = pd.read_feather(root/'train.feather')
 test_df = pd.read_feather(root/'test.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather' )
INPUT_SIZE = X_train[feature_name].shape[1]
OUTPUT_SIZE = 199
print(INPUT_SIZE)
 BATCH_SIZE = 512
EPOCHS = 50
def get_model(x_tr, y_tr, x_val, y_val):
model = Sequential()
model.add(Dense(1024, input_shape=(INPUT_SIZE,)))
 model.add(Activation('relu'))
model.add(Dropout(0.7))
model.add(BatchNormalization())
 model.add(Dense(512, input_shape=(INPUT_SIZE,)))
 model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(BatchNormalization())
 model.add(Dense(384))
model.add(Activation('relu'))
model.add(Dropout(0.35))
model.add(BatchNormalization())
 model.add(Dense(256))
model.add(Activation('relu'))
model.add(Dropout(0.3))
model.add(BatchNormalization())
 model.add(Dense(OUTPUT_SIZE))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='Adam', metrics=[])
 es = EarlyStopping(monitor='CRPS_score_val',
mode='min',
restore_best_weights=True,
verbose=1,
patience=12)
 mc = ModelCheckpoint('best_model.h5',monitor='CRPS_score_val',mode='min',
save_best_only=True, verbose=1, save_weights_only=True)
 model.fit(x_tr, y_tr,
batch_size = BATCH_SIZE,
epochs=EPOCHS,
callbacks = [CRPSCallback(validation =(x_val, y_val)) , es, mc],
verbose=1)
 y_pred = model.predict(x_val)
 y_valid = y_val
y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * x_val.shape[0])
 crps = np.round(val_s, 6)
 return model ,crps<train_on_grid>
leak_df = pd.read_feather('.. /input/ashrae-leak-data-station/leak.feather')
 leak_df.fillna(0, inplace=True)
 leak_df = leak_df[(leak_df.timestamp.dt.year > 2016)&(leak_df.timestamp.dt.year < 2019)]
leak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0
leak_df = leak_df[leak_df.building_id!=245]
losses = []
models = []
crps_csv = []
kfold = KFold(5, shuffle=True, random_state=42)
 for k_fold,(tr_inds, val_inds)in enumerate(kfold.split(yards_train)) :
print(len(tr_inds), len(val_inds))
tr_x,tr_y = X_train.iloc[tr_inds][feature_name],y_train[tr_inds]
val_x,val_y = X_train.iloc[val_inds][feature_name],y_train[val_inds]
model,crps = get_model(tr_x,tr_y,val_x,val_y)
 models.append(model)
 print("the %d fold crps is %f"%(( k_fold+1),crps))
crps_csv.append(crps)
 print("mean crps is %f"%np.mean(crps_csv))
print(len(models))<predict_on_test>
leak_df.meter.value_counts()
def predict(x_te):
model_num = len(models)
 for k,m in enumerate(models):
if k==0:
y_pred = m.predict(x_te,batch_size=BATCH_SIZE)
 else:
y_pred+=m.predict(x_te,batch_size=BATCH_SIZE)
 y_pred = y_pred / model_num
return y_pred<categorify>
print(leak_df.duplicated().sum() )
env = nflrush.make_env()
iter_test = env.iter_test()
row_count = 0
for(test_df, sample_prediction_df)in iter_test:
basetable = get_data(test_df, mode='test')
 for c in basetable.columns:
if(basetable[c].dtype != int):
basetable[c] = basetable[c].astype('float32')
 for c in basetable.columns:
if basetable[c].isnull().any() :
print(c)
 print(basetable[c].mean())
 basetable[c] = basetable[c].fillna(0)
 basetable.drop(['PlayId', 'GameId'], axis=1, inplace=True)
 scaled_basetable = scaler.transform(basetable)
 scaled_basetable = pd.DataFrame(scaled_basetable, index=basetable.index, columns=basetable.columns)
 y_pred = predict(scaled_basetable[feature_name])
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1 ).tolist() [0]
preds_df = pd.DataFrame(data=[y_pred], columns=sample_prediction_df.columns)
 env.predict(preds_df)
 row_count += 1
env.write_submission_file() 
<set_options>
print(len(leak_df)/ len(train_df))
pd.options.display.max_rows = 4000
warnings.filterwarnings('ignore')
 sns.set()
%matplotlib inline
pd.set_option('display.max_columns', 50)
 pd.set_option('display.max_rows', 150 )<categorify>
del train_df
gc.collect()
def clean_WindDirection(txt):
if isinstance(txt, int):
return('NA')
 elif pd.isna(txt):
return('NA')
 else:
if pd.isna(txt):
return np.nan
txt = txt.lower()
txt = ''.join([c for c in txt if c not in punctuation])
 txt = txt.replace('from', '')
 txt = txt.replace(' ', '')
 txt = txt.replace('north', 'n')
 txt = txt.replace('south', 's')
 txt = txt.replace('west', 'w')
 txt = txt.replace('east', 'e')
 return txt
def transform_WindDirection(txt):
if pd.isna(txt):
return('-1')
 if txt=='n':
return 0
elif txt=='nne' or txt=='nen':
return 1/8
elif txt=='ne':
return 2/8
elif txt=='ene' or txt=='nee':
return 3/8
elif txt=='e':
return 4/8
elif txt=='ese' or txt=='see':
return 5/8
elif txt=='se':
return 6/8
elif txt=='ses' or txt=='sse':
return 7/8
elif txt=='s':
return 8/8
elif txt=='ssw' or txt=='sws':
return 9/8
elif txt=='sw':
return 10/8
elif txt=='sww' or txt=='wsw':
return 11/8
elif txt=='w':
return 12/8
elif txt=='wnw' or txt=='nww':
return 13/8
elif txt=='nw':
return 14/8
elif txt=='nwn' or txt=='nnw':
return 15/8
else: return -1
return np.nan
def clean_StadiumType(txt):
if pd.isna(txt):
return('NA')
 if isinstance(txt, int):
return('NA')
 else:
txt = txt.lower()
txt = ''.join([c for c in txt if c not in punctuation])
 txt = re.sub(' +', ' ', txt)
 txt = txt.strip()
txt = txt.replace('outside', 'outdoor')
 txt = txt.replace('outdor', 'outdoor')
 txt = txt.replace('outddors', 'outdoor')
 txt = txt.replace('outdoors', 'outdoor')
 txt = txt.replace('oudoor', 'outdoor')
 txt = txt.replace('indoors', 'indoor')
 txt = txt.replace('ourdoor', 'outdoor')
 txt = txt.replace('retractable', 'rtr.')
 return txt
def set_Turf(df):
Turf = {'Field Turf':'Artificial', 'A-Turf Titan':'Artificial', 'Grass':'Natural',
'UBU Sports Speed S5-M':'Artificial', 'Artificial':'Artificial',
'DD GrassMaster':'Artificial', 'Natural Grass':'Natural', 'UBU Speed Series-S5-M':'Artificial',
'FieldTurf':'Artificial', 'FieldTurf 360':'Artificial', 'Natural grass':'Natural', 'grass':'Natural',
'Natural':'Natural', 'Artifical':'Artificial', 'FieldTurf360':'Artificial', 'Naturall Grass':'Natural',
'Field turf':'Artificial', 'SISGrass':'Artificial', 'Twenty-Four/Seven Turf':'Artificial',
'natural grass':'Natural'}
df['Turf_type']=df['Turf'].map(Turf)
 return df
def GameWeather(df):
if isinstance(df['GameWeather'].iloc[0], np.integer):
df['GameWeather']='NA'
return(df)
 else:
df['GameWeather'] = df['GameWeather'].str.lower()
indoor = "indoor"
df['GameWeather'] = df['GameWeather'].apply(lambda x: indoor if not pd.isna(x)and indoor in x else x)
 df['GameWeather'] = df['GameWeather'].apply(lambda x: x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly')if not pd.isna(x)else x)
 df['GameWeather'] = df['GameWeather'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear')if not pd.isna(x)else x)
 df['GameWeather'] = df['GameWeather'].apply(lambda x: x.replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 return df<feature_engineering>
sample_submission1 = pd.read_csv('.. /input/ashrae-kfold-lightgbm-without-leak-1-08/submission.csv', index_col=0)
 sample_submission2 = pd.read_csv('.. /input/ashrae-half-and-half/submission.csv', index_col=0)
 sample_submission3 = pd.read_csv('.. /input/ashrae-highway-kernel-route4/submission.csv', index_col=0)
 sample_submission4 = pd.read_csv('.. /input/stratifiedkfoldlgbxopy/submission.csv', index_col=0 )
def clean_TeamAbbreviations(train_df):
train_df.loc[train_df.VisitorTeamAbbr == "ARI", 'VisitorTeamAbbr'] = "ARZ"
train_df.loc[train_df.HomeTeamAbbr == "ARI", 'HomeTeamAbbr'] = "ARZ"
train_df.loc[train_df.VisitorTeamAbbr == "BAL", 'VisitorTeamAbbr'] = "BLT"
train_df.loc[train_df.HomeTeamAbbr == "BAL", 'HomeTeamAbbr'] = "BLT"
train_df.loc[train_df.VisitorTeamAbbr == "CLE", 'VisitorTeamAbbr'] = "CLV"
train_df.loc[train_df.HomeTeamAbbr == "CLE", 'HomeTeamAbbr'] = "CLV"
train_df.loc[train_df.VisitorTeamAbbr == "HOU", 'VisitorTeamAbbr'] = "HST"
train_df.loc[train_df.HomeTeamAbbr == "HOU", 'HomeTeamAbbr'] = "HST"
return train_df<feature_engineering>
test_df['pred1'] = sample_submission1.meter_reading
test_df['pred2'] = sample_submission2.meter_reading
test_df['pred3'] = sample_submission3.meter_reading
test_df['pred4'] = sample_submission4.meter_reading
test_df.loc[test_df.pred3<0, 'pred3'] = 0
test_df.loc[test_df.pred3<0, 'pred4'] = 0
del sample_submission1, sample_submission2, sample_submission3, sample_submission4
gc.collect()
test_df = reduce_mem_usage(test_df)
 leak_df = reduce_mem_usage(leak_df )
def tag_rusher(train_df):
train_df['IsBallCarrier'] = train_df.NflId == train_df.NflIdRusher
return train_df<feature_engineering>
leak_df = leak_df.merge(test_df[['building_id', 'meter', 'timestamp', 'pred1', 'pred2', 'pred3', 'pred4', 'row_id']], left_on = ['building_id', 'meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'], how = "left")
 leak_df = leak_df.merge(building_meta_df[['building_id', 'site_id']], on='building_id', how='left' )
def get_direction(train_df):
train_df['ToLeft'] = train_df.PlayDirection == "left"
return train_df<feature_engineering>
leak_df['pred1_l1p'] = np.log1p(leak_df.pred1)
 leak_df['pred2_l1p'] = np.log1p(leak_df.pred2)
 leak_df['pred3_l1p'] = np.log1p(leak_df.pred3)
 leak_df['pred4_l1p'] = np.log1p(leak_df.pred4)
 leak_df['meter_reading_l1p'] = np.log1p(leak_df.meter_reading )
def feet_inch_to_cm(height_fi):
foot = 30.48
inch = 2.54
return int(height_fi.split('-')[0])* foot + int(height_fi.split('-')[1])* inch
def imperial_to_metric(train_df):
train_df['PlayerHeight_cm']= train_df['PlayerHeight'].apply(feet_inch_to_cm)
 train_df['PlayerWeight_kg']= train_df['PlayerWeight']*0.453592
return train_df<feature_engineering>
leak_df[leak_df.pred1_l1p.isnull() ]
def get_player_age(train_df):
train_df['Age']=2019 - pd.DatetimeIndex(train_df['PlayerBirthDate'] ).year
return train_df<categorify>
leak_df['mean_pred'] = np.mean(leak_df[['pred1', 'pred2', 'pred3', 'pred4']].values, axis=1)
 leak_df['mean_pred_l1p'] = np.log1p(leak_df.mean_pred)
 leak_score = np.sqrt(mean_squared_error(leak_df.mean_pred_l1p, leak_df.meter_reading_l1p))
sns.distplot(leak_df.mean_pred_l1p)
 sns.distplot(leak_df.meter_reading_l1p)
 print('mean score=', leak_score )
def possession_team(possessionteam, hometeam, visitorteam):
if possessionteam == hometeam:
return 'home'
else:
return 'visitor'
def leading_team_home_visitor(homescore, visitorscore):
if homescore == visitorscore:
return 'draw'
else:
if homescore > visitorscore:
return 'home'
else:
return 'visitor'
def leading_team_def_off(team, posessionteam, hometeam, visitorteam, homescore, visitorscore):
if homescore == visitorscore:
return 'draw'
else:
if posessionteam == hometeam:
if homescore > visitorscore:
return 'offense'
else:
return 'defense'
else:
if visitorscore > homescore:
return 'offense'
else:
return 'defense'
def get_leading_team_features(train_df):
train_df['possession_team_home_visitor'] = train_df.apply(lambda row: possession_team(row.PossessionTeam,row.HomeTeamAbbr,row.VisitorTeamAbbr), axis = 1)
 train_df['leading_by'] = abs(train_df['HomeScoreBeforePlay'] - train_df['VisitorScoreBeforePlay'])
 train_df['leading_team_defense_offense'] = train_df.apply(lambda row: leading_team_def_off(row.Team,
row.PossessionTeam,
row.HomeTeamAbbr,
row.VisitorTeamAbbr,
row.HomeScoreBeforePlay,
row.VisitorScoreBeforePlay), axis = 1)
 train_df['leading_team_home_visitor'] = train_df.apply(lambda row: leading_team_home_visitor(row.HomeScoreBeforePlay,row.VisitorScoreBeforePlay), axis = 1)
 train_df['Offense_score']=np.where(train_df['leading_team_defense_offense']=='offense', train_df['leading_by'], -train_df['leading_by'])
 train_df['Deffense_score']=np.where(train_df['leading_team_defense_offense']=='defense', train_df['leading_by'], -train_df['leading_by'])
 train_df['Offense_Team'] = train_df['PossessionTeam']
train_df['Defense_Team'] = train_df.apply(lambda row: row['HomeTeamAbbr'] if row['PossessionTeam'] != row['HomeTeamAbbr'] else row['VisitorTeamAbbr'],axis=1)
 return train_df<feature_engineering>
class GAOptimizer:
def __init__(self, function, min_value=0.2, max_value=0.8, population_size=50, dimention=4):
self.function = function
self.population = np.random.uniform(min_value, max_value,(population_size, dimention))
self.population_size = population_size
self.dimention = dimention
half_dim1 = int(dimention/2)
 half_dim2 = int(dimention/2)+ 1 if dimention%2 else int(dimention/2)
 self.co_weights = np.hstack([np.ones(( population_size, half_dim1)) , np.zeros(( population_size, half_dim2)) ])
 def crossover(self):
old_population = np.copy(self.population)
 population = np.copy(self.population)
 np.random.shuffle(population)
 new_population = old_population*self.co_weights + population*(1 - self.co_weights)+ np.random.normal(0, 0.1,(self.population_size, self.dimention))
return np.vstack([old_population, new_population])
 def selector(self, n=None):
f_values = self.function(self.population)
 self.population = self.population[np.argsort(f_values)[:self.population_size]]
if n:
return self.population[np.argsort(f_values)[:n]], self.function(self.population[np.argsort(f_values)[:n]])
 def fit(self, iters):
for i in range(iters):
self.population = self.crossover()
self.selector()
return self.selector(1 )
def get_adjusted_yardline(train_df):
train_df['Home_team_play']=np.where(train_df['PossessionTeam']==train_df['HomeTeamAbbr'], 1, 0)
 train_df['Team_side']=np.where(train_df['PossessionTeam']==train_df['FieldPosition'], 1, 0)
 train_df['YardLine_adj']=np.where(train_df['Team_side']==1,train_df['YardLine'],100-train_df['YardLine'])
 return train_df<data_type_conversions>
def func_to_opt(scores):
score = []
for x in scores:
v = x[0] * leak_df['pred1'].values + x[1] * leak_df['pred2'].values + \
x[2] * leak_df['pred3'].values + x[3] * leak_df['pred4'].values
val =(v > 0 ).astype(int)*v
vl1p = np.log1p(val)
 curr_score = np.sqrt(mean_squared_error(vl1p, leak_df.meter_reading_l1p))
score.append(curr_score)
 return np.array(score )
def get_GameClock_features(train_df):
train_df['snap_to_handoff']=(pd.to_datetime(train_df['TimeHandoff'])-pd.to_datetime(train_df['TimeSnap'])).dt.total_seconds()
sec=pd.to_timedelta(train_df['GameClock'])
 sec=sec.dt.total_seconds() /60
train_df['GameClock_sec']=sec
train_df['clock_bin']=np.where(train_df['Quarter']==1,1,np.where(train_df['Quarter']==2,2,np.where(train_df['Quarter']==3,3,
np.where(train_df['Quarter']==4,np.where(train_df['GameClock_sec']<=564,4,5),np.where(train_df['GameClock_sec']<=333,6,7)))))
 return train_df<string_transform>
%%time
ga_holder = GAOptimizer(function=lambda l: func_to_opt(l))
result = ga_holder.fit(40)
 print(result )
def get_personnel_types(dataset):
setups = list(dataset['OffensePersonnel'].unique())
 setups = ','.join(setups)
 setups = re.sub(r'\d+', '', setups ).replace(' ','')
 mylist = list(set(setups.split(',')))
 offense_column_names = ['Offense_' + s for s in mylist]
setups = list(dataset['DefensePersonnel'].unique())
 setups = ','.join(setups)
 setups = re.sub(r'\d+', '', setups ).replace(' ','')
 mylist = list(set(setups.split(',')))
 defense_column_names = ['Defense_' + s for s in mylist]
return offense_column_names + defense_column_names
def extract_personnel_count(whole_personnel, specific_personnel):
specific_personnel = specific_personnel.split('_')[1]
if specific_personnel in whole_personnel:
whole_personnel = whole_personnel.split(',')
 types = [re.sub(r'\d+', '', x ).replace(' ','')for x in whole_personnel]
counts = [[int(s)for s in x.split() if s.isdigit() ][0] for x in whole_personnel]
dictionary = dict(zip(types, counts))
return dictionary[specific_personnel]
else:
return 0
def get_split_personnel_types(columns_to_add, train_df):
for col in columns_to_add:
if col.split('_')[0] == 'Defense':
train_df[col] = train_df['DefensePersonnel'].apply(lambda x: extract_personnel_count(x, col))
else:
train_df[col] = train_df['OffensePersonnel'].apply(lambda x: extract_personnel_count(x, col))
return train_df 
<groupby>
final = result[0].flatten() /result[0].flatten().sum()
print(final )
def add_bins_and_generate_stats(train_df):
bins=[-99,-0.01,0.01,1.72,4.21,6.7,99]
train_df['Yards_Bin'] = pd.cut(train_df['Yards'], bins)
 stats = {}
for my_type in ['DisplayName', 'Offense_Team', 'Defense_Team']:
per_my_type = train_df[[my_type,'Yards','Yards_Bin']].groupby([my_type,'Yards_Bin'] ).count().fillna(0 ).reset_index()
per_my_type = per_my_type.merge(per_my_type.groupby(my_type)['Yards'].sum().reset_index() , on=my_type, how='left')
 per_my_type['Yards_Bin_Rate'] = per_my_type['Yards_x'] / per_my_type['Yards_y']
per_my_type.rename(columns={'Yards_x':'Bin_Count', 'Yards_y': 'Total_Plays'}, inplace=True)
 per_my_type = pd.pivot_table(per_my_type, values=['Bin_Count', 'Yards_Bin_Rate'], index=[my_type], columns=['Yards_Bin'])
 per_my_type.columns = per_my_type.columns.map('{0[0]}_{0[1]}'.format)
 per_my_type = per_my_type.reset_index()
avg_yards = train_df.groupby(my_type)['Yards'].mean().reset_index()
std_yards = train_df.groupby(my_type)['Yards'].std().reset_index().fillna(0)
 counts = train_df.groupby(my_type)['Yards'].count().reset_index()
per_type = avg_yards.merge(counts, on=my_type, how='inner' ).merge(std_yards, on=my_type, how='inner')
 per_type.rename(columns={'Yards_x':'Avg_Yards', 'Yards_y':'Count', 'Yards':'Std_Yards'}, inplace= True)
 per_my_type= per_my_type.merge(per_type, on=my_type, how='left')
 if my_type == 'DisplayName':
per_player_enough_plays = per_type[per_type['Count'] >= 10]
per_player_not_enough_plays = per_type[per_type['Count'] < 10]
per_player_not_enough_plays['rusher_performance_bin'] = 'few_plays'
per_player_enough_plays['rusher_performance_bin'] = pd.qcut(per_player_enough_plays['Avg_Yards'], q=[0, 0.2,0.5,0.9,1], labels= ['bottom_20','20-50','50-90','top_10'])
 per_player = pd.concat([per_player_enough_plays, per_player_not_enough_plays])
 per_my_type= per_my_type.merge(per_player[[my_type,'rusher_performance_bin']], on=my_type, how='left')
 per_my_type.columns = [per_my_type.columns[0]] + [my_type + '_' + str(col)for col in per_my_type.columns[1:]]
stats[my_type] = per_my_type
train_df = train_df.merge(per_my_type, on=my_type, how='left')
 return stats, train_df<data_type_conversions>
func_to_opt(np.array([final]))
def convert_to_cat(df):
for col in ['DefensePersonnel','Home_team_play','Team_side','Down','Defense_Team','OffenseFormation','OffensePersonnel',
'leading_team_defense_offense','PlayDirection','clock_bin','Defense_DB','Defense_LB','Defense_DL','Offense_WR']:
df[col] = df[col].astype('category')
 return df<compute_test_metric>
sample_submission = pd.read_feather(os.path.join(root, 'sample_submission.feather'))
w1 = final[0]
w2 = final[1]
w3 = final[2]
w4 = final[3]
sample_submission['meter_reading'] = w1 * test_df.pred1 + w2 * test_df.pred2 + w3 * test_df.pred3 +\
w4 * test_df.pred4
sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0
def euclidean_distance(x1,x2,y1,y2):
return np.sqrt(( x1-x2)**2 +(y1-y2)**2)
 def norm_x(df):
df['Player_o_d']=np.where(( df["Home_team_play"] ==1)&(df["Team"] == 'home')|(df["Home_team_play"] ==0)&(df["Team"] == 'away'),'Offense_Player','Defense_Player')
 df['YardLine_Direction']=np.where(df["PlayDirection"] =='right',df['YardLine_adj'],100-df['YardLine_adj'])
 df['X_adj']=np.where(df["Player_o_d"] =='Offense_Player',abs(df['YardLine_Direction']+10-df['X']),-abs(df['YardLine_Direction']+10-df['X']))
return(df)
 def qb_and_rusher_to_closest_defender(df):
play_ids = list(df.PlayId.unique())
 rows = []
for play_id in play_ids:
play = df[df['PlayId'] == play_id]
rusher = play[play['IsBallCarrier']==True][['X', 'Y']]
QB_offense=play[play['Position']=='QB'][['X','Y']]
defenders = play[play['Player_o_d']=='Defense_Player'][['X', 'Y']]
defenders['rusher_X'] = rusher['X'].iloc[0]
defenders['rusher_Y'] = rusher['Y'].iloc[0]
try:
defenders['qb_X'] = QB_offense['X'].iloc[0]
defenders['qb_Y'] = QB_offense['Y'].iloc[0]
except:
defenders['qb_X'] = 50
defenders['qb_Y'] = 50
defenders['dist_rusher_to_defender'] = defenders.apply(lambda row: euclidean_distance(row.X,row.rusher_X,row.Y,row.rusher_Y), axis = 1)
 defenders['dist_qb_to_defender'] = defenders.apply(lambda row: euclidean_distance(row.X,row.qb_X,row.Y,row.qb_Y), axis = 1)
 rows.append([play_id, defenders['dist_rusher_to_defender'].min() , defenders['dist_qb_to_defender'].min() , euclidean_distance(rusher['X'].iloc[0], defenders['qb_X'].iloc[0], rusher['Y'].iloc[0], defenders['qb_Y'].iloc[0])])
 rows = pd.DataFrame(rows, columns=['PlayId', 'min_dist_rusher_to_defender', 'min_dist_qb_to_defender', 'dist_rusher_QB'])
 df = df.merge(rows, on='PlayId', how='left')
 return df
def Direction_orientation_adj(df):
df['dir_adj']=np.where(df["Player_o_d"] =='Offense_Player',np.where(df["PlayDirection"] =='right',df["Dir"],360-df["Dir"]),np.where(df["PlayDirection"] =='right',360-df["Dir"],df["Dir"]))
df['Orientation_adj']=np.where(df["Player_o_d"] =='Offense_Player',np.where(df["PlayDirection"] =='right',df["Orientation"],360-df["Orientation"]),np.where(df["PlayDirection"] =='right',360-df["Orientation"],df["Orientation"]))
df['dir_to_Orientation']=180 - abs(abs(df['dir_adj'] - df['Orientation_adj'])- 180)
 return(df)
 def X_Y_Velocity(df):
df['v_x']=np.sin(df['dir_adj'] * np.pi/180)*df['S']
df['v_y']=np.cos(df['dir_adj'] * np.pi/180)*df['S']
df['arc_tan']=np.arctan(( df['v_y']/df['v_x'])) *180/np.pi
df['arc_tan_cat']=pd.cut(df['arc_tan'],bins=[-90, -45, 0, 45, 90],labels=False)
 df['t1_distance']=df['Dis']+df['S']*1+0.5*df['A']*1
df['t2_distance']=df['Dis']+df['S']*2+0.5*df['A']*4
df['t3_distance']=df['Dis']+df['S']*3+0.5*df['A']*9
df['v_x_a1']=np.sin(df['dir_adj'] * np.pi/180)*(df['S']+df['A']*1)
 df['v_y_a1']=np.cos(df['dir_adj'] * np.pi/180)*(df['S']+df['A']*1)
 df['v_x_a2']=np.sin(df['dir_adj'] * np.pi/180)*(df['S']+df['A']*2)
 df['v_y_a2']=np.cos(df['dir_adj'] * np.pi/180)*(df['S']+df['A']*2)
 df['v_x_a3']=np.sin(df['dir_adj'] * np.pi/180)*(df['S']+df['A']*3)
 df['v_y_a3']=np.cos(df['dir_adj'] * np.pi/180)*(df['S']+df['A']*3)
 df['a_x']=np.sin(df['dir_adj'] * np.pi/180)*df['A']
df['a_y']=np.cos(df['dir_adj'] * np.pi/180)*df['A']
return(df )<set_options>
leak_df = leak_df[['meter_reading', 'row_id']].set_index('row_id' ).dropna()
sample_submission.loc[leak_df.index, 'meter_reading'] = leak_df['meter_reading']
h2o.init(min_mem_size='16G' )<train_model>
sample_submission.isna().sum()
<load_from_csv><EOS> 
sample_submission.to_csv('submission.csv', index=False, float_format='%.5f' )