Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<drop_column>
if not sys.warnoptions:
warnings.simplefilter("ignore" )
train_df = train_df.sort_values(by=['PlayId', 'Team', 'IsRusher', 'JerseyNumber'] ).reset_index()
train_df = train_df.drop(['index', 'IsRusher', 'Team'], axis=1)
 train_df = train_df.set_index(['GameId', 'PlayId'])
 cat_features = []
for col in train_df.columns:
if train_df[col].dtype =='object':
cat_features.append(col)
 train_df = train_df.drop(cat_features, axis=1)
 train_df.fillna(-999, inplace=True)
 clean_time = time.time()
print("Finish Data Cleaning, session time: {}s".format(clean_time - start_time))<categorify>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
def clean_data(df):
df['StadiumType'] = df['StadiumType'].apply(clean_StadiumType)
 df['StadiumType'] = df['StadiumType'].apply(transform_StadiumType)
 df['DefendersInTheBox_vs_Distance'] = df['DefendersInTheBox'] / df['Distance']
df['OffenseFormation'] = df['OffenseFormation'].apply(lambda x: x if x in off_form else np.nan)
 df = pd.concat([df.drop(['OffenseFormation'], axis=1), pd.get_dummies(df['OffenseFormation'], prefix='Formation')], axis=1)
 missing_cols = set(dummy_col)- set(df.columns)-set('Yards')
 for c in missing_cols:
df[c] = 0
df = df[dummy_col]
df['Turf'] = df['Turf'].map(Turf)
 df['Turf'] = df['Turf'] == 'Natural'
df['PossessionTeam'] = df['PossessionTeam'].map(map_abbr)
 df['HomeTeamAbbr'] = df['HomeTeamAbbr'].map(map_abbr)
 df['VisitorTeamAbbr'] = df['VisitorTeamAbbr'].map(map_abbr)
 df['HomePossesion'] = df['PossessionTeam'] == df['HomeTeamAbbr']
df['Field_eq_Possession'] = df['FieldPosition'] == df['PossessionTeam']
df['HomeField'] = df['FieldPosition'] == df['HomeTeamAbbr']
df['GameClock'] = df['GameClock'].apply(strtoseconds)
 df['PlayerHeight'] = df['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
df['PlayerBMI'] = 703*(df['PlayerWeight']/(df['PlayerHeight'])**2)
 df['TimeHandoff'] = df['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeSnap'] = df['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeDelta'] = df.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 df['PlayerBirthDate'] = df['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60*60*24*365.25
df['PlayerAge'] = df.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1)
 df['WindSpeed'] = df['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x)
 df['WindSpeed'] = df['WindSpeed'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 df['WindSpeed'] = df['WindSpeed'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x)
 df['WindSpeed'] = df['WindSpeed'].apply(lambda x: float(x)if str(x ).isnumeric() else -1)
 df['WindDirection'] = df['WindDirection'].apply(clean_WindDirection)
 df['WindDirection'] = df['WindDirection'].apply(transform_WindDirection)
 df['PlayDirection'] = df['PlayDirection'].apply(lambda x: x.strip() == 'right')
 df['Team'] = df['Team'].apply(lambda x: x.strip() =='home')
 indoor = "indoor"
df['GameWeather'] = df['GameWeather'].apply(lambda x: indoor if not pd.isna(x)and indoor in x else x)
 df['GameWeather'] = df['GameWeather'].apply(lambda x: x.lower().replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly' ).replace('clear and sunny', 'sunny and clear' ).replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 df['GameWeather'] = df['GameWeather'].apply(map_weather)
 df['IsRusher'] = df['NflId'] == df['NflIdRusher']
df['X'] = df.apply(lambda row: row['X'] if row['PlayDirection'] else 120-row['X'], axis=1)
 df['Orientation'] = df.apply(lambda row: new_orientation(row['Orientation'], row['PlayDirection']), axis=1)
 df['Dir'] = df.apply(lambda row: new_orientation(row['Dir'], row['PlayDirection']), axis=1)
 df['YardsLeft'] = df.apply(lambda row: 100-row['YardLine'] if row['HomeField'] else row['YardLine'], axis=1)
 df['YardsLeft'] = df.apply(lambda row: row['YardsLeft'] if row['PlayDirection'] else 100-row['YardsLeft'], axis=1)
 df = df.sort_values(by=['PlayId', 'Team', 'IsRusher', 'JerseyNumber'] ).reset_index()
df = df.drop(['TimeHandoff', 'TimeSnap', 'PlayerBirthDate', 'NflId', 'NflIdRusher', 'index', 'IsRusher', 'Team'], axis=1)
 df = df.set_index(['GameId', 'PlayId'])
 cat_features = []
for col in df.columns:
if df[col].dtype =='object':
cat_features.append(col)
 df = df.drop(cat_features, axis=1)
 df.fillna(-999, inplace=True)
 return df 
<import_modules>
train_ID = train['Id']
test_ID = test['Id']
train.drop("Id", axis = 1, inplace = True)
 test.drop("Id", axis = 1, inplace = True )
from sklearn.preprocessing import StandardScaler
import torch
import torch.nn as nn
import torchvision<categorify>
train["SalePrice"] = np.log1p(train["SalePrice"] )
def transform_df(df, is_train=True, col1=play_cols, col2=player_cols):
if is_train:
play_cols.remove('Yards')
 play_df = df[play_cols]
player_df = df[player_cols]
targets = df['Yards']
play_df = play_df[::22]
x1 = torch.tensor(StandardScaler().fit_transform(play_df ).astype(float), dtype=torch.float32)
 player_df = StandardScaler().fit_transform(player_df)
 player_df = player_df.reshape(play_df.shape[0], player_df.shape[1], 22)
 x2 = torch.tensor(player_df.astype(float), dtype=torch.float32)
 index = play_df.reset_index("GameId" ).index
y = torch.tensor(targets[::22].values, dtype=torch.float32)
 y_dis = torch.zeros([y.shape[0], 199])
 y_dis[torch.arange(y.shape[0]),(y + 99 ).long() ] = 1
return x1, x2, y_dis, index
else:
play_df = df[play_cols]
player_df = df[player_cols]
play_df = play_df[::22]
x1 = torch.tensor(StandardScaler().fit_transform(play_df ).astype(float), dtype=torch.float32)
 player_df = StandardScaler().fit_transform(player_df)
 player_df = player_df.reshape(play_df.shape[0], player_df.shape[1], 22)
 x2 = torch.tensor(player_df.astype(float), dtype=torch.float32)
 index = play_df.reset_index("GameId" ).index
return x1, x2, index
x1, x2, y, index = transform_df(train_df )<choose_model_class>
ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train.SalePrice.values
all_data = pd.concat(( train, test)).reset_index(drop=True)
 all_data.drop(['SalePrice'], axis=1, inplace=True)
 print("all_data size is: {}".format(all_data.shape))
class MixtureNN(nn.Module):
def __init__(self):
super(MixtureNN, self ).__init__()
self.x1_dense = nn.Sequential(
 nn.Linear(len(play_cols), 200),
nn.ReLU() ,
nn.Linear(200, 100),
nn.ReLU() , 
)
 self.x2_conv = nn.Sequential(
 nn.Conv1d(in_channels=len(player_cols), out_channels=22, kernel_size=5, stride=1, padding=1),
nn.ReLU() ,
nn.Conv1d(in_channels=22, out_channels=11, kernel_size=3, stride=1, padding=1),
nn.ReLU() , 
)
 self.x_dense = nn.Sequential(
 nn.Linear(320, 480),
nn.BatchNorm1d(480),
nn.ReLU() ,
nn.Linear(480, 240),
nn.BatchNorm1d(240),
nn.ReLU() ,
nn.Linear(240, 199),
nn.Softmax(dim=1)
)
 def forward(self, x1, x2):
x1 = self.x1_dense(x1)
 x2 = self.x2_conv(x2 ).reshape(x1.shape[0], 220)
 x = torch.cat(( x1, x2), dim=1)
 x = self.x_dense(x)
 return x<compute_test_metric>
all_data_missing =(all_data.isnull().sum() / len(all_data)) * 100
all_data_missing = all_data_missing.drop(all_data_missing[all_data_missing == 0].index ).sort_values(ascending=False)
 missing_data = pd.DataFrame({'Missing Percentage':all_data_missing})
 missing_data.head(30 )
MixtureNN()(x1,x2 ).shape<split>
for col in('PoolQC', 'MiscFeature', 'Alley', 'Fence', 'FireplaceQu', 'GarageType', 'GarageFinish', 'GarageQual', 'GarageCond', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'MasVnrType', 'MSSubClass'):
all_data[col] = all_data[col].fillna('None' )
before_training_time = time.time()
loss_func = CRPS()
def random_split(val_rate=0.2, batch=512):
total_index = np.arange(len(y))
val_index = np.random.choice(total_index, int(len(total_index)* 0.2), replace=False)
 train_index = np.delete(total_index, val_index)
 train_set = torch.utils.data.TensorDataset(x1[train_index], x2[train_index], y[train_index])
 val_set = torch.utils.data.TensorDataset(x1[val_index], x2[val_index], y[val_index])
 train_loader = torch.utils.data.DataLoader(train_set, batch_size=batch, shuffle=True, num_workers=8)
 val_loader = torch.utils.data.DataLoader(val_set, batch_size=batch, shuffle=True, num_workers=8)
 return train_loader, val_loader
def calculate_average_loss(dataloader: torch.utils.data.DataLoader):
running_loss = 0
for x1_batch, x2_batch, y_batch in dataloader:
prediction = model(x1_batch, x2_batch)
 loss = loss_func(prediction, y_batch)
 running_loss += float(loss)
 return running_loss / len(dataloader.dataset)
 print("======== START TRAINING...... ")
 cv_times = 0
models_params = []
sum_best_loss = 0
while cv_times < 20:
stable_step = 0
best_val_loss = 100
epoch = 0
session_time = time.time()
model = MixtureNN()
model.train()
optimizer = torch.optim.Adam(model.parameters() , lr=0.0001, weight_decay=0.01)
 print("======== Start Model {} ========".format(cv_times))
train_loader, val_loader = random_split(val_rate=0.3, batch=512)
 while stable_step < 50:
for x1_batch, x2_batch, y_batch in train_loader:
output = model(x1_batch, x2_batch)
 loss_val = loss_func(output, y_batch)
 loss_val.backward()
optimizer.step()
train_ave_loss = calculate_average_loss(train_loader)
 val_ave_loss = calculate_average_loss(val_loader)
 if val_ave_loss < best_val_loss:
best_val_loss = val_ave_loss
best_params = model.state_dict()
stable_step = 0
else:
stable_step += 1
epoch += 1
print("epoch: {}, train_loss: {}, val_loss: {}, best_val_loss: {}, stable_step: {}".format(
 epoch, train_ave_loss, val_ave_loss, best_val_loss, stable_step))
if epoch >= 500:
break
print("======== Finish Model {} ========".format(cv_times))
print("Model training time: {}s.The best evaluation loss is {}.".format(time.time() - session_time, best_val_loss))
models_params.append(best_params)
 sum_best_loss += best_val_loss
cv_times += 1
training_time = time.time()
print("======== END, total training time: {}s".format(training_time - before_training_time))
print("average_best_loss: {}".format(sum_best_loss / cv_times))<define_variables>
for col in('GarageYrBlt', 'GarageArea', 'GarageCars', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath', 'MasVnrArea'):
all_data[col] = all_data[col].fillna(0 )
TOTAL: {}s".format(time.time() - init_time))<predict_on_test>
for col in('MSZoning', 'Electrical', 'KitchenQual', 'Exterior1st', 'Exterior2nd', 'SaleType', 'Functional', 'Utilities'):
all_data[col] = all_data[col].fillna(all_data[col].mode() [0] )
def make_pred(df, sample, env, models_params):
clean_df = df
clean_df = clean_data(clean_df)
 x1, x2, index = transform_df(clean_df, is_train=False)
 y_preds = []
for params in models_params:
model.load_state_dict(params)
 model.eval()
y_preds.append(torch.cumsum(model(x1, x2), dim=1 ).detach().data.numpy())
 y_ave = np.array(y_preds ).mean(0)
 yardsleft = np.array(clean_df["YardsLeft"][::22])
 for i in range(len(yardsleft)) :
y_ave[i, :yardsleft[i] - 1] = 0
y_ave[i, yardsleft[i] + 99:] = 1
env.predict(pd.DataFrame(data=y_ave.clip(0, 1), columns=sample.columns))
return y_ave<predict_on_test>
all_data['LotFrontage'] = all_data.groupby('Neighborhood')['LotFrontage'].apply(lambda x: x.fillna(x.median()))
for test, sample in tqdm.tqdm(env.iter_test()):
make_pred(test, sample, env, models_params)
<save_to_csv>
all_data_missing =(all_data.isnull().sum() / len(all_data)) * 100
all_data_missing = all_data_missing.drop(all_data_missing[all_data_missing == 0].index ).sort_values(ascending=False)
 missing_data = pd.DataFrame({'Missing Ratio':all_data_missing})
 missing_data.head(30 )
env.write_submission_file()<train_model>
for col in('MSSubClass', 'OverallCond', 'YrSold', 'MoSold'):
all_data[col] = all_data[col].astype(str)
 all_data.info()
total_time = time.time()
print('Total running time is: {}s'.format(total_time - start_time))<categorify>
skewness = skewness[abs(skewness)> 0.75]
print("Total number of features requiring a fix for skewness is: {}".format(skewness.shape[0]))
warnings.simplefilter(action='ignore', category=FutureWarning)
 env = nflrush.make_env()
categories = []
labelEncoder = LabelEncoder()
rushers =[]
in_out_map = defaultdict(int,{
"open": 1,
"field": 1,
"out": 1,
"oud": 1,
"our": 1,
"cloud": 1,
"close": 0,
"retract": 0,
"dome": 0,
})
 weather_map = {
"controlled": 1,
"indoor": 1,
"indoors": 1,
"indoors": 1,
"sunny": 0.8,
"clear": 0.6,
"cloudy": 0.4,
"coudy": 0.4,
"hazy": 0.4,
"cool": 0.4,
"rain": 0.2,
"rainy": 0.2,
"cold": 0,
"snow": 0,
}
turf_map = {
"Field Turf": "Artificial",
"A-Turf Titan": "Artificial",
"Grass": "Natural",
"UBU Sports Seed S5-M": "Artificial",
"Artificial": "Artificial",
"DD GrassMaster": "Artificial",
"Natural Grass": "Natural",
"UBU Seed Series-S5-M": "Artificial",
"FieldTurf": "Artificial",
"FieldTurf 360": "Artificial",
"Natural grass": "Natural",
"grass": "Natural",
"Natural": "Natural",
"Artifical": "Artificial",
"FieldTurf360": "Artificial",
"Naturall Grass": "Natural",
"Field turf": "Artificial",
"SISGrass": "Artificial",
"Twenty-Four/Seven Turf": "Artificial",
"natural grass": "Natural"
}
train_df = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2020/train.csv', low_memory=False )<categorify>
skewed_features = skewness.index
lam = 0.15
for feature in skewed_features:
all_data[feature] = boxcox1p(all_data[feature], lam )
def windSpeedToInt(windSpeed):
x = re.findall('[0-9]*', str(windSpeed))
if len(x)> 1 and x[0].isdigit() and x[1].isdigit() :
return(int(x[0])+ int(x[1])) / 2
if not x[0].isdigit() :
return np.nan
return(x[0])
 def stadiumTypeToInt(stadium):
x = re.findall(r"[\w']+", str(stadium ).lower())
 weight = 0
if len(x)== 0:
return np.nan
for word in x:
weight += in_out_map[word]
weight = weight / len(x)
 return round(weight)
 def heightToInches(height):
return int(height.split('-')[0])* 12 + int(height.split('-')[1])
 def convertToProb(num):
return np.array([1 if i > num + 99 else 0 for i in range(199)])
 def crps(y_true, y_pred):
y_pred = np.clip(np.cumsum(y_pred,axis = 1),0,1)
 return np.mean(( y_pred-y_true)**2)
 def gameWeatherToInt(weather):
x = re.findall(r"[\w']+", str(weather ).lower())
 valid = 0
weight = 0
for word in x:
if word in weather_map:
weight += weather_map[word]
valid += 1
if valid == 0:
return np.nan
return str(weight / valid)
 def gameClockToSeconds(gameClock):
times = str(gameClock ).split(':')
 return(int(( int(times[0])* 60)+ int(times[1])* int(times[2])/ 60))
def cleanData(df):
df = df.replace(["ARZ","BLT","CLV","HST"],["ARI","BAL","CLE","HOU"])
 df["WindSpeed"] = df["WindSpeed"].apply(windSpeedToInt ).astype("float64")
 df["PlayerHeight"] = df["PlayerHeight"].apply(heightToInches)
 df["StadiumType"] = df["StadiumType"].apply(stadiumTypeToInt ).astype("category")
 df["GameWeather"] = df["GameWeather"].apply(gameWeatherToInt ).astype("category")
 df['TeamOnOffense'] = "home"
df.loc[df.PossessionTeam != df.HomeTeamAbbr, 'TeamOnOffense'] = "away"
df['IsOnOffense'] = df.Team == df.TeamOnOffense
df['MovingLeft'] = df.PlayDirection == "left"
df['StdOrientation'] = df.Orientation
df.loc[df.MovingLeft, 'StdOrientation'] = np.mod(180 + df.loc[df.MovingLeft, 'StdOrientation'], 360)
 df['StdX'] = df.X
df.loc[df.MovingLeft, 'StdX'] = 120 - df.loc[df.MovingLeft, 'X']
df['StdY'] = df.Y
df.loc[df.MovingLeft, 'StdY'] = 53.3 - df.loc[df.MovingLeft, 'Y']
df['StdDir'] = df.Dir
df.loc[df.MovingLeft, 'StdDir'] = np.mod(180 + df.loc[df.MovingLeft, 'StdDir'], 360)
 df.loc[df['Season'] == 2017, 'Orientation'] = np.mod(90 + df.loc[df['Season'] == 2017, 'Orientation'], 360)
 df['StdYardline'] = 100 - df.YardLine
df.loc[df.FieldPosition.fillna('')== df.PossessionTeam,'StdYardline'] = df.loc[df.FieldPosition.fillna('')== df.PossessionTeam,'YardLine']
df['Turf'] = df['Turf'].map(turf_map)
 df['GameClock_seconds'] = train_df['GameClock'].apply(gameClockToSeconds)
 df['GameClock_minutes'] = train_df['GameClock'].apply(lambda time: int(time.split(':')[0]))
df['TimeHandoff'] = df['TimeHandoff'].apply(lambda time: datetime.datetime.strptime(time, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeSnap'] = df['TimeSnap'].apply(lambda time: datetime.datetime.strptime(time, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeDelta'] = df.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 df = df.drop(["X","Y","Dir","Orientation","YardLine","TimeHandoff","TimeSnap"],axis=1)
 return df<categorify>
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
def pipeline(train_df):
train_df = cleanData(train_df)
 train_df["WindDirection"] = train_df["WindDirection"].fillna("Unknown")
 train_df["FieldPosition"] = train_df["FieldPosition"].fillna("Unknown")
 train_df["OffenseFormation"] = train_df["OffenseFormation"].fillna("Unknown")
 train_df["Turf"] = train_df["Turf"].fillna("Unknown")
 categorical_features = train_df.select_dtypes(include=["object"] ).columns
global labelEncoder
global categories
global rushers
if len(categories)== 0:
for feature in categorical_features:
train_df[feature] = feature + "__" + train_df[feature].astype("str")
 categories += train_df[feature].unique().tolist()
categories += [feature +"__Unknown"]
labelEncoder.fit(categories)
 for feature in categorical_features:
train_df[feature] = labelEncoder.transform(train_df[feature])
 rushers = train_df[train_df["NflId"] == train_df["NflIdRusher"]]
rushers = rushers.groupby(["NflId"])["Yards"].agg(AvgCarry="mean")
 else:
for feature in categorical_features:
train_df[feature] = feature + "__" + train_df[feature].astype("str")
 unseenLabels = ~train_df[feature].isin(categories)
 train_df.loc[unseenLabels,[feature]] = feature + "__Unknown"
train_df[feature] = labelEncoder.transform(train_df[feature])
 train_df["NearestOffensivePlayer"] = 0
train_df["NearestDefensivePlayer"] = 0
for play in train_df.PlayId.unique() :
playXY = train_df.loc[train_df["PlayId"] == play, ['StdX', 'StdY', 'NflId', 'NflIdRusher', 'PossessionTeam', 'Team', 'HomeTeamAbbr',"Position"]]
rusher_team = playXY.loc[playXY['NflId'] == playXY['NflIdRusher'], ['Team']]
rusherXY = playXY.loc[playXY['NflId'] == playXY['NflIdRusher'], ['StdX', 'StdY']]
playersXY = playXY.loc[playXY['NflId'] != playXY['NflIdRusher'], ['StdX', 'StdY', 'Team',"Position"]]
playerOffenseXY = playersXY.loc[playersXY['Team'] == rusher_team.iloc[0]['Team'], ['StdX', 'StdY',"Position"]]
playerDefenseXY = playersXY.loc[playersXY['Team'] != rusher_team.iloc[0]['Team'], ['StdX', 'StdY',"Position"]]
o_neighbours = NearestNeighbors(n_neighbors=1, algorithm='ball_tree' ).fit(playerOffenseXY.drop(["Position"],axis=1))
d_neighbours = NearestNeighbors(n_neighbors=1, algorithm='ball_tree' ).fit(playerDefenseXY.drop(["Position"],axis=1))
o_dist, _ = o_neighbours.kneighbors(rusherXY)
 d_dist, _ = d_neighbours.kneighbors(rusherXY)
 train_df.loc[train_df['PlayId'] == play, 'NearestOffensivePlayer'] = o_dist[0][0]
train_df.loc[train_df['PlayId'] == play, 'NearestDefensivePlayer'] = d_dist[0][0]
playerOLBXY = playerDefenseXY.loc[playerDefenseXY["Position"] == labelEncoder.transform(['Position__OLB'])[0], ['StdX', 'StdY']]
playerILBXY = playerDefenseXY.loc[playerDefenseXY["Position"] == labelEncoder.transform(['Position__ILB'])[0], ['StdX', 'StdY']]
playerDEXY = playerDefenseXY.loc[playerDefenseXY["Position"] == labelEncoder.transform(['Position__DE'])[0], ['StdX', 'StdY']]
playerDTXY = playerDefenseXY.loc[playerDefenseXY["Position"] == labelEncoder.transform(['Position__DT'])[0], ['StdX', 'StdY']]
playersTackleXY = pd.concat([playerOLBXY,playerILBXY,playerDEXY,playerDTXY])
 if not playersTackleXY.empty:
tackle_neighbours = NearestNeighbors(n_neighbors=1, algorithm='ball_tree' ).fit(playersTackleXY)
 dis, _ = tackle_neighbours.kneighbors(rusherXY)
 else:
dis = [[1000000]]
train_df.loc[train_df['NflId'] == play, 'NearestLikelyTackle'] = dis[0][0]
train_df[categorical_features] = train_df[categorical_features].astype('category')
 train_df = train_df[train_df["NflId"] == train_df["NflIdRusher"]]
train_df["hspeed"] = train_df["S"] * np.sin(np.deg2rad(train_df["StdDir"]))
train_df["to_line"] = train_df["StdYardline"] - train_df["StdX"]
train_df["force"] = train_df["A"] * train_df["PlayerWeight"]
train_df = pd.merge(train_df,rushers, on="NflId",how="left")
 train_df["AvgCarry"] = train_df["AvgCarry"].fillna(0)
 dropCols = categorical_features.drop(["Team","OffenseFormation","OffensePersonnel","DefensePersonnel","WindDirection"])
 dropCols = dropCols.append(pd.Index(["NflIdRusher","Season","WindSpeed","GameId","PlayId","Week"]))
train_df = train_df.drop(dropCols, axis=1)
 return train_df<prepare_x_and_y>
low_var_cat = [col for col in all_data.select_dtypes(exclude=['number'])if 1 - sum(all_data[col] == mode(all_data[col])) /len(all_data)< 0.03]
low_var_cat
def train_my_model(train_df):
train_df = pipeline(train_df)
 columns = list(train_df.columns)
 columns.remove("Yards")
 X_train = train_df[columns]
Y_train = train_df["Yards"]
params = { 'objective' : 'multiclass', 'num_classes': 199,
'num_leaves': 19, 'feature_fraction': 0.4,
'subsample': 0.4, 'min_child_samples': 10, 'num_threads': 5,
'learning_rate': 0.01, 'num_iterations': 100, 'random_state': 42}
bestScore = 1000
bestModel = ""
kfold = KFold(n_splits=3, shuffle=True, random_state=42)
 for train_index, test_index in kfold.split(X_train, Y_train):
X_train_fold = X_train.iloc[train_index]
Y_train_fold = Y_train.iloc[train_index]
X_test_fold = X_train.iloc[test_index]
Y_test_fold = Y_train.iloc[test_index]
Y_train_fold = Y_train_fold + 99
dataSet = lgb.Dataset(X_train_fold,label=Y_train_fold)
 model = lgb.train(params,dataSet)
 Y_pred = model.predict(X_test_fold)
 Y_test_fold = np.vstack(pd.Series(Y_test_fold ).apply(lambda x: convertToProb(x)))
 score = crps(Y_test_fold,Y_pred)
 if score < bestScore:
bestModel = copy.deepcopy(model)
 bestScore = score
print(score)
 return bestModel<predict_on_test>
all_data = all_data.drop(['Street', 'Utilities', 'Condition2', 'RoofMatl', 'Heating', 'PoolQC'], axis=1 )
def predict(test_df,model):
test_df = pipeline(test_df)
 return pd.DataFrame(np.clip(np.cumsum(model.predict(test_df),axis = 1),0,1),columns=sample_prediction_df.columns)
 for(test_df, sample_prediction_df)in env.iter_test() :
predictions_df = predict(test_df, model)
 env.predict(predictions_df)
 env.write_submission_file()<set_options>
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(all_data[c].values))
all_data[c] = lbl.transform(list(all_data[c].values))
print('Shape all_data: {}'.format(all_data.shape))
tqdm.pandas() 
<import_modules>
all_data = pd.get_dummies(all_data)
 all_data.shape
from kaggle.competitions import nflrush<set_options>
train = all_data[:ntrain]
test = all_data[ntrain:]
def init_seeds(seed):
np.random.seed(seed)
 rn.seed(seed)
 tf.set_random_seed(seed)
 sess = tf.Session(graph=tf.get_default_graph())
 K.set_session(sess)
 return sess<load_from_csv>
X_train = train
X_test = test
def rmse_cv(model):
rmse = np.sqrt(-cross_val_score(model, X_train, y_train, scoring="neg_mean_squared_error", cv = 10))
return(rmse )
input_folder = '.. /input/nfl-big-data-bowl-2020'
train = pd.read_csv(os.path.join(input_folder, 'train.csv'), dtype={'WindSpeed': 'object'})
 train.shape
results = {}<define_variables>
alphas = [0.05, 0.1, 0.3, 1, 3, 5, 10, 15, 30]
cv_ridge = [rmse_cv(Ridge(alpha = alpha)).mean() for alpha in alphas]
player_cols = ['X', 'Y', 'X_dir', 'Y_dir', 'X_S', 'Y_S', 'S', 'A', 'IsRusher', 'IsOnOffense']
play_cols = ['X_rusher', 'Y_rusher', 'YardLine_std']<feature_engineering>
model_ridge = Ridge(alpha = 5 )
def reorient(train, flip_left):
train['ToLeft'] = train.PlayDirection == "left"
train.loc[train.VisitorTeamAbbr == "ARI", 'VisitorTeamAbbr'] = "ARZ"
train.loc[train.HomeTeamAbbr == "ARI", 'HomeTeamAbbr'] = "ARZ"
train.loc[train.VisitorTeamAbbr == "BAL", 'VisitorTeamAbbr'] = "BLT"
train.loc[train.HomeTeamAbbr == "BAL", 'HomeTeamAbbr'] = "BLT"
train.loc[train.VisitorTeamAbbr == "CLE", 'VisitorTeamAbbr'] = "CLV"
train.loc[train.HomeTeamAbbr == "CLE", 'HomeTeamAbbr'] = "CLV"
train.loc[train.VisitorTeamAbbr == "HOU", 'VisitorTeamAbbr'] = "HST"
train.loc[train.HomeTeamAbbr == "HOU", 'HomeTeamAbbr'] = "HST"
train['TeamOnOffense'] = "home"
train.loc[train.PossessionTeam != train.HomeTeamAbbr, 'TeamOnOffense'] = "away"
train['IsOnOffense'] = train.Team == train.TeamOnOffense
train['YardLine_std'] = 100 - train.YardLine
train.loc[train.FieldPosition.fillna('')== train.PossessionTeam,
'YardLine_std'
] = train.loc[train.FieldPosition.fillna('')== train.PossessionTeam,
'YardLine']
train.loc[train.ToLeft, 'X'] = 120 - train.loc[train.ToLeft, 'X']
train.loc[train.ToLeft, 'Y'] = 160/3 - train.loc[train.ToLeft, 'Y']
train.loc[train.ToLeft, 'Orientation'] = np.mod(180 + train.loc[train.ToLeft, 'Orientation'], 360)
 train['Dir'] = 90 - train.Dir
train.loc[train.ToLeft, 'Dir'] = np.mod(180 + train.loc[train.ToLeft, 'Dir'], 360)
 train.loc[train.IsOnOffense, 'Dir'] = train.loc[train.IsOnOffense, 'Dir'].fillna(0 ).values
train.loc[~train.IsOnOffense, 'Dir'] = train.loc[~train.IsOnOffense, 'Dir'].fillna(180 ).values
train['S'] = 10 * train['Dis']
train.loc[train.Season == 2017, 'A'] = train.loc[train.Season == 2017, 'A'].mean()
train['IsRusher'] = train['NflId'] == train['NflIdRusher']
if flip_left:
df = train[train['IsRusher']].copy()
df['left'] = df.Dir < 0
train = train.merge(df[['PlayId', 'left']], how='left', on='PlayId')
 train['Y'] = train.Y
train.loc[train.left, 'Y'] = 160/3 - train.loc[train.left, 'Y']
train['Dir'] = train.Dir
train.loc[train.left, 'Dir'] = np.mod(- train.loc[train.left, 'Dir'], 360)
 train.drop('left', axis=1, inplace=True)
 train['X_dir'] = np.cos(( np.pi / 180)* train.Dir)
 train['Y_dir'] = np.sin(( np.pi / 180)* train.Dir)
 train['X_S'] = train.X_dir * train.S
train['Y_S'] = train.Y_dir * train.S
train['X_A'] = train.X_dir * train.A
train['Y_A'] = train.Y_dir * train.A
train['PlayerHeight'] = train['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
train['BMI'] =(train['PlayerWeight'] * 703)/(train['PlayerHeight'] ** 2)
 train['Energy'] = train['PlayerWeight'] *(train['S'] ** 2)
 train['time_step'] = 0.0
train = train.sort_values(by=['PlayId', 'IsOnOffense', 'IsRusher', 'Y'] ).reset_index(drop=True)
 return train
def add_features(train):
df = train[train.IsRusher][['PlayId', 'time_step', 'X', 'Y']].copy()
df.columns = ['PlayId', 'time_step', 'X_rusher', 'Y_rusher']
train = train.merge(df, how='left', on=['PlayId', 'time_step'])
 train.X -= train.X_rusher
train.Y -= train.Y_rusher
return train
def time_forward(train, time_step):
train = train.copy()
train['X'] = train['X'] + time_step * train['X_S'] + 0.5 * time_step**2 * train['X_A']
train['Y'] = train['Y'] + time_step * train['Y_S'] + 0.5 * time_step**2 * train['Y_A']
x_s = np.clip(train['X_S'] + time_step * train['X_A'], -14, 14)
 y_s = np.clip(train['Y_S'] + time_step * train['Y_A'], -14, 14)
 if 1:
train['X_A'] = np.clip(train['X_A'] * train['X_S'] /(x_s), -10, 10)
 train['Y_A'] = np.clip(train['Y_A'] * train['Y_S'] /(y_s), -10, 10)
 train['X_S'] = x_s
train['Y_S'] = y_s
train['time_step'] = time_step
return train<concatenate>
model_lasso = make_pipeline(RobustScaler() , Lasso(alpha = 0.0005))
train_reorient = reorient(train.copy() , flip_left=True)
 len_train_reorient = len(train_reorient)// 22<concatenate>
model_elastic = make_pipeline(RobustScaler() , ElasticNet(alpha = 0.0005))
def augment_data(train_reorient, time_steps):
train_reorient = pd.concat(time_forward(train_reorient, time_step)for time_step in time_steps)
 train_reorient = add_features(train_reorient)
 return train_reorient<categorify>
model_krr = make_pipeline(RobustScaler() , KernelRidge(alpha=6, kernel='polynomial', degree=2.65, coef0=6.9))
time_steps = [0, 0.3, 0.6]
train_reorient = augment_data(train_reorient, time_steps )<define_variables>
model_gbr = GradientBoostingRegressor(n_estimators=1200,
learning_rate=0.05,
max_depth=4,
max_features='sqrt',
min_samples_leaf=15,
min_samples_split=10,
loss='huber',
random_state=5 )
len_2017 = train[train.Season == 2017].shape[0] // 22
len_2017<prepare_x_and_y>
model_xgb = xgb.XGBRegressor(colsample_bytree=0.2,
learning_rate=0.06,
max_depth=3,
n_estimators=1150 )
ss_players = StandardScaler()
X_players = df.values.astype('float32')
 X_players = ss_players.fit_transform(df)
 X_players = X_players.reshape(( -1, 22, len(player_cols)))
 X_players.shape<normalization>
model_lgb = lgb.LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.05,
n_estimators=1080,
max_bin=75,
bagging_fraction=0.80,
bagging_freq=5,
feature_fraction=0.232,
feature_fraction_seed=9,
bagging_seed=9,
min_data_in_leaf=6,
min_sum_hessian_in_leaf=11 )
X_play = train_reorient[play_cols].values[::22]
ss_play = StandardScaler()
X_play = ss_play.fit_transform(X_play )<compute_test_metric>
model_cat = CatBoostRegressor(iterations=2000,
learning_rate=0.10,
depth=3,
l2_leaf_reg=4,
border_count=15,
loss_function='RMSE',
verbose=200)
 params = {'iterations':2000,
'learning_rate':0.10,
'depth':3,
'l2_leaf_reg':4,
'border_count':15,
'loss_function':'RMSE',
'verbose':200}
@jit
def get_dmat(X, Y):
dmat = np.zeros(( 22, 22))
for i in range(22):
for j in range(i+1, 22):
d = np.sqrt(( X[i] - X[j])**2 +(Y[i] - Y[j])**2)
 dmat[i, j] = d
dmat[j, i] = d
dmat = dmat.reshape(( 1, 22, 22))
return dmat<prepare_x_and_y>
cv_ridge = rmse_cv(model_ridge ).mean()
cv_lasso = rmse_cv(model_lasso ).mean()
cv_elastic = rmse_cv(model_elastic ).mean()
cv_krr = rmse_cv(model_krr ).mean()
cv_gbr = rmse_cv(model_gbr ).mean()
cv_xgb = rmse_cv(model_xgb ).mean()
cv_lgb = rmse_cv(model_lgb ).mean()
X,Y = train_reorient.X.values, train_reorient.Y.values
dmats = [get_dmat(X[i:i+22], Y[i:i+22])for i in range(0, train_reorient.shape[0], 22)]
dmats = np.vstack(dmats ).reshape(( -1, 22, 22))
dmats.shape<drop_column>
pool = Pool(X_train, y_train)
 cv_cat = cv(pool=pool, params=params, fold_count=10, shuffle=True )
del train<choose_model_class>
results = pd.DataFrame({
'Model': ['Ridge',
'Lasso',
'ElasticNet',
'Kernel Ridge',
'Gradient Boosting Regressor',
'XGBoost Regressor',
'Light Gradient Boosting Regressor',
'CatBoost'],
'Score': [cv_ridge,
cv_lasso,
cv_elastic,
cv_krr,
cv_gbr,
cv_xgb,
cv_lgb,
cv_cat]})
 result_df = results.sort_values(by='Score', ascending=True ).reset_index(drop=True)
 result_df.head(8 )
class LayerNormalization(keras.layers.Layer):
def __init__(self,
center=True,
scale=True,
epsilon=None,
gamma_initializer='ones',
beta_initializer='zeros',
gamma_regularizer=None,
beta_regularizer=None,
gamma_constraint=None,
beta_constraint=None,
**kwargs):

 super(LayerNormalization, self ).__init__(**kwargs)
 self.supports_masking = True
self.center = center
self.scale = scale
if epsilon is None:
epsilon = K.epsilon() * K.epsilon()
self.epsilon = epsilon
self.gamma_initializer = keras.initializers.get(gamma_initializer)
 self.beta_initializer = keras.initializers.get(beta_initializer)
 self.gamma_regularizer = keras.regularizers.get(gamma_regularizer)
 self.beta_regularizer = keras.regularizers.get(beta_regularizer)
 self.gamma_constraint = keras.constraints.get(gamma_constraint)
 self.beta_constraint = keras.constraints.get(beta_constraint)
 self.gamma, self.beta = None, None
def get_config(self):
config = {
'center': self.center,
'scale': self.scale,
'epsilon': self.epsilon,
'gamma_initializer': keras.initializers.serialize(self.gamma_initializer),
'beta_initializer': keras.initializers.serialize(self.beta_initializer),
'gamma_regularizer': keras.regularizers.serialize(self.gamma_regularizer),
'beta_regularizer': keras.regularizers.serialize(self.beta_regularizer),
'gamma_constraint': keras.constraints.serialize(self.gamma_constraint),
'beta_constraint': keras.constraints.serialize(self.beta_constraint),
}
base_config = super(LayerNormalization, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
def compute_output_shape(self, input_shape):
return input_shape
def compute_mask(self, inputs, input_mask=None):
return input_mask
def build(self, input_shape):
shape = input_shape[-1:]
if self.scale:
self.gamma = self.add_weight(
 shape=shape,
initializer=self.gamma_initializer,
regularizer=self.gamma_regularizer,
constraint=self.gamma_constraint,
name='gamma', 
)
 if self.center:
self.beta = self.add_weight(
 shape=shape,
initializer=self.beta_initializer,
regularizer=self.beta_regularizer,
constraint=self.beta_constraint,
name='beta', 
)
 super(LayerNormalization, self ).build(input_shape)
 def call(self, inputs, training=None):
mean = K.mean(inputs, axis=-1, keepdims=True)
 variance = K.mean(K.square(inputs - mean), axis=-1, keepdims=True)
 std = K.sqrt(variance + self.epsilon)
 outputs =(inputs - mean)/ std
if self.scale:
outputs *= self.gamma
if self.center:
outputs += self.beta
return outputs<prepare_x_and_y>
model_lasso.fit(X_train, y_train)
 lasso_pred = np.expm1(model_lasso.predict(X_test))
model_elastic.fit(X_train, y_train)
 elastic_pred = np.expm1(model_elastic.predict(X_test))
model_ridge.fit(X_train, y_train)
 ridge_pred = np.expm1(model_ridge.predict(X_test))
model_xgb.fit(X_train, y_train)
 xgb_pred = np.expm1(model_xgb.predict(X_test))
model_gbr.fit(X_train, y_train)
 gbr_pred = np.expm1(model_gbr.predict(X_test))
model_lgb.fit(X_train, y_train)
 lgb_pred = np.expm1(model_lgb.predict(X_test))
model_krr.fit(X_train, y_train)
 krr_pred = np.expm1(model_krr.predict(X_test))
model_cat.fit(X_train, y_train)
 cat_pred = np.expm1(model_cat.predict(X_test))
y_mae = train_reorient['Yards'].values[::22]
y_mean = np.median(y_mae)
 y_crps = np.zeros(shape=(y_mae.shape[0], 199))
for i,yard in enumerate(y_mae):
y_crps[i, yard+99:] = 1
yardline = train_reorient['YardLine_std'].values[::22]<compute_test_metric>
stacked =(lasso_pred + elastic_pred + ridge_pred + xgb_pred + lgb_pred + krr_pred + gbr_pred)/ 7
<data_type_conversions><EOS> 
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = stacked
sub.to_csv('house_price_predictions.csv',index=False )