Kaggle competition: NFL Big Data Bowl
X_valid_train.drop('date_block_num', axis=1, inplace=True)
 X_valid_test.drop('date_block_num', axis=1, inplace=True)
 X_train.drop('date_block_num', axis=1, inplace=True)
 X_test.drop('date_block_num', axis=1, inplace=True )<compute_train_metric>
import datetime
import matplotlib.pyplot as plt
import re
from string import punctuation
import time
import tqdm
model = LinearRegression()
model.fit(X_valid_train, y_valid_train)
 pred = model.predict(X_valid_test ).clip(0,20)
 rmse = np.sqrt(mean_squared_error(y_valid_test, pred))
print('RMSE on valid set: {0}'.format(rmse))<train_model>
env = nflrush.make_env()
model = LinearRegression()
model.fit(X_train, y_train)
 pred = model.predict(X_test ).clip(0,20 )<save_to_csv>
start_time = time.time()
original_df = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2020/train.csv', low_memory=False)
 train_df = original_df
train_df.head()
submit = pd.DataFrame({'ID':range(len(pred)) , 'item_cnt_month': pred})
 submit.to_csv('submit.csv', index=False )<load_from_csv>
train_df['DefendersInTheBox_vs_Distance'] = train_df['DefendersInTheBox'] / train_df['Distance']
items = pd.read_csv('.. /input/items.csv')
 shops = pd.read_csv('.. /input/shops.csv')
 cats = pd.read_csv('.. /input/item_categories.csv')
 train = pd.read_csv('.. /input/sales_train.csv')
 test = pd.read_csv('.. /input/test.csv' ).set_index('ID' )<filter>
def clean_StadiumType(txt):
if pd.isna(txt):
return np.nan
txt = txt.lower()
txt = ''.join([c for c in txt if c not in punctuation])
 txt = re.sub(' +', ' ', txt)
 txt = txt.strip()
txt = txt.replace('outside', 'outdoor')
 txt = txt.replace('outdor', 'outdoor')
 txt = txt.replace('outddors', 'outdoor')
 txt = txt.replace('outdoors', 'outdoor')
 txt = txt.replace('oudoor', 'outdoor')
 txt = txt.replace('indoors', 'indoor')
 txt = txt.replace('ourdoor', 'outdoor')
 txt = txt.replace('retractable', 'rtr.')
 return txt
def transform_StadiumType(txt):
if pd.isna(txt):
return np.nan
if 'outdoor' in txt or 'open' in txt:
return 1
if 'indoor' in txt or 'closed' in txt:
return 0
return np.nan
train_df["StadiumType"] = train_df["StadiumType"].apply(clean_StadiumType)
 train_df['StadiumType'] = train_df['StadiumType'].apply(transform_StadiumType )
train = train[train.item_price<100000]
train = train[train.item_cnt_day<1001]<feature_engineering>
Turf = {'Field Turf':'Artificial', 'A-Turf Titan':'Artificial', 'Grass':'Natural', 'UBU Sports Speed S5-M':'Artificial',
'Artificial':'Artificial', 'DD GrassMaster':'Artificial', 'Natural Grass':'Natural',
'UBU Speed Series-S5-M':'Artificial', 'FieldTurf':'Artificial', 'FieldTurf 360':'Artificial', 'Natural grass':'Natural', 'grass':'Natural',
'Natural':'Natural', 'Artifical':'Artificial', 'FieldTurf360':'Artificial', 'Naturall Grass':'Natural', 'Field turf':'Artificial',
'SISGrass':'Artificial', 'Twenty-Four/Seven Turf':'Artificial', 'natural grass':'Natural'}
train_df['Turf'] = train_df['Turf'].map(Turf)
 train_df['Turf'] = train_df['Turf'] == 'Natural'
median = train[(train.shop_id==32)&(train.item_id==2973)&(train.date_block_num==4)&(train.item_price>0)].item_price.median()
train.loc[train.item_price<0, 'item_price'] = median<feature_engineering>
map_abbr = {'ARI': 'ARZ', 'BAL': 'BLT', 'CLE': 'CLV', 'HOU': 'HST'}
for abb in train_df['PossessionTeam'].unique() :
map_abbr[abb] = abb
train_df['PossessionTeam'] = train_df['PossessionTeam'].map(map_abbr)
 train_df['HomeTeamAbbr'] = train_df['HomeTeamAbbr'].map(map_abbr)
 train_df['VisitorTeamAbbr'] = train_df['VisitorTeamAbbr'].map(map_abbr)
 train_df['HomePossesion'] = train_df['PossessionTeam'] == train_df['HomeTeamAbbr']
train_df['Field_eq_Possession'] = train_df['FieldPosition'] == train_df['PossessionTeam']
train_df['HomeField'] = train_df['FieldPosition'] == train_df['HomeTeamAbbr']
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 10, 'shop_id'] = 11
test.loc[test.shop_id == 10, 'shop_id'] = 11<categorify>
off_form = train_df['OffenseFormation'].unique()
train_df = pd.concat([train_df.drop(['OffenseFormation'], axis=1), pd.get_dummies(train_df['OffenseFormation'], prefix='Formation')], axis=1)
 dummy_col = train_df.columns
shops.loc[shops.shop_name == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
shops['city'] = shops['shop_name'].str.split(' ' ).map(lambda x: x[0])
 shops.loc[shops.city == '!Якутск', 'city'] = 'Якутск'
shops['city_code'] = LabelEncoder().fit_transform(shops['city'])
 shops = shops[['shop_id','city_code']]
cats['split'] = cats['item_category_name'].str.split('-')
 cats['type'] = cats['split'].map(lambda x: x[0].strip())
 cats['type_code'] = LabelEncoder().fit_transform(cats['type'])
 cats['subtype'] = cats['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats['subtype_code'] = LabelEncoder().fit_transform(cats['subtype'])
 cats = cats[['item_category_id','type_code', 'subtype_code']]
items.drop(['item_name'], axis=1, inplace=True )<concatenate>
def strtoseconds(txt):
txt = txt.split(':')
 ans = int(txt[0])*60 + int(txt[1])+ int(txt[2])/60
return ans
train_df['GameClock'] = train_df['GameClock'].apply(strtoseconds )
len(list(set(test.item_id)- set(test.item_id ).intersection(set(train.item_id)))) , len(list(set(test.item_id))), len(test )<data_type_conversions>
def strtoinches(txt):
txt = txt.split('-')
 ans = int(txt[0])* 12 + int(txt[1])
 return ans
train_df['PlayerHeight'] = train_df['PlayerHeight'].apply(strtoinches)
 train_df['PlayerBMI'] = 703 *(train_df['PlayerWeight']/(train_df['PlayerHeight'])**2 )
ts = time.time()
matrix = []
cols = ['date_block_num','shop_id','item_id']
for i in range(34):
sales = train[train.date_block_num==i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype='int16'))
matrix = pd.DataFrame(np.vstack(matrix), columns=cols)
 matrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)
 matrix['shop_id'] = matrix['shop_id'].astype(np.int8)
 matrix['item_id'] = matrix['item_id'].astype(np.int16)
 matrix.sort_values(cols,inplace=True)
 time.time() - ts<feature_engineering>
train_df['TimeHandoff'] = train_df['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train_df['TimeSnap'] = train_df['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train_df['TimeDelta'] = train_df.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1 )
train['revenue'] = train['item_price'] * train['item_cnt_day']<merge>
train_df['PlayerBirthDate'] = train_df['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60*60*24*365.25
train_df['PlayerAge'] = train_df.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() / seconds_in_year, axis=1)
 train_df = train_df.drop(['TimeHandoff', 'TimeSnap', 'PlayerBirthDate'], axis=1 )
ts = time.time()
group = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day': ['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=cols, how='left')
 matrix['item_cnt_month'] =(matrix['item_cnt_month'] 
.fillna(0)
.clip(0,20)
.astype(np.float16))
time.time() - ts<data_type_conversions>
train_df['WindSpeed'] = train_df['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x)
 train_df['WindSpeed'] = train_df['WindSpeed'].apply(lambda x:(int(x.split('-')[0])+ int(x.split('-')[1])) / 2 if not pd.isna(x)and '-' in x else x)
 train_df['WindSpeed'] = train_df['WindSpeed'].apply(lambda x:(int(x.split() [0])+ int(x.split() [-1])) / 2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x)
 train_df['WindSpeed'] = train_df['WindSpeed'].apply(lambda x: float(x)if str(x ).isnumeric() else -1)
 def clean_WindDirection(txt):
if pd.isna(txt):
return np.nan
txt = txt.lower()
txt = ''.join([c for c in txt if c not in punctuation])
 txt = txt.replace('from', '')
 txt = txt.replace(' ', '')
 txt = txt.replace('north', 'n')
 txt = txt.replace('south', 's')
 txt = txt.replace('west', 'w')
 txt = txt.replace('east', 'e')
 return txt
def transform_WindDirection(txt):
if pd.isna(txt):
return np.nan
if txt=='n':
return 0
if txt=='nne' or txt=='nen':
return 1/8
if txt=='ne':
return 2/8
if txt=='ene' or txt=='nee':
return 3/8
if txt=='e':
return 4/8
if txt=='ese' or txt=='see':
return 5/8
if txt=='se':
return 6/8
if txt=='ses' or txt=='sse':
return 7/8
if txt=='s':
return 8/8
if txt=='ssw' or txt=='sws':
return 9/8
if txt=='sw':
return 10/8
if txt=='sww' or txt=='wsw':
return 11/8
if txt=='w':
return 12/8
if txt=='wnw' or txt=='nww':
return 13/8
if txt=='nw':
return 14/8
if txt=='nwn' or txt=='nnw':
return 15/8
return np.nan
train_df['WindDirection'] = train_df['WindDirection'].apply(clean_WindDirection)
 train_df['WindDirection'] = train_df['WindDirection'].apply(transform_WindDirection )
test['date_block_num'] = 34
test['date_block_num'] = test['date_block_num'].astype(np.int8)
 test['shop_id'] = test['shop_id'].astype(np.int8)
 test['item_id'] = test['item_id'].astype(np.int16 )<concatenate>
train_df['PlayDirection'] = train_df['PlayDirection'].apply(lambda x: x.strip() == 'right' )
ts = time.time()
matrix = pd.concat([matrix, test], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace=True)
 time.time() - ts<data_type_conversions>
train_df['Team'] = train_df['Team'].apply(lambda x: x.strip() =='home' )
ts = time.time()
matrix = pd.merge(matrix, shops, on=['shop_id'], how='left')
 matrix = pd.merge(matrix, items, on=['item_id'], how='left')
 matrix = pd.merge(matrix, cats, on=['item_category_id'], how='left')
 matrix['city_code'] = matrix['city_code'].astype(np.int8)
 matrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)
 matrix['type_code'] = matrix['type_code'].astype(np.int8)
 matrix['subtype_code'] = matrix['subtype_code'].astype(np.int8)
 time.time() - ts<merge>
train_df['GameWeather'] = train_df['GameWeather'].str.lower()
indoor = "indoor"
train_df['GameWeather'] = train_df['GameWeather'].apply(lambda x: indoor if not pd.isna(x)and indoor in x else x)
 train_df['GameWeather'] = train_df['GameWeather'].apply(lambda x: x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly')if not pd.isna(x)else x)
 train_df['GameWeather'] = train_df['GameWeather'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear')if not pd.isna(x)else x)
 train_df['GameWeather'] = train_df['GameWeather'].apply(lambda x: x.replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 def map_weather(txt):
ans = 1
if pd.isna(txt):
return 0
if 'partly' in txt:
ans*=0.5
if 'climate controlled' in txt or 'indoor' in txt:
return ans*3
if 'sunny' in txt or 'sun' in txt:
return ans*2
if 'clear' in txt:
return ans
if 'cloudy' in txt:
return -ans
if 'rain' in txt or 'rainy' in txt:
return -2*ans
if 'snow' in txt:
return -3*ans
return 0
train_df['GameWeather'] = train_df['GameWeather'].apply(map_weather )
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df<merge>
train_df['IsRusher'] = train_df['NflId'] == train_df['NflIdRusher']
train_df.drop(['NflId', 'NflIdRusher'], axis=1, inplace=True )
ts = time.time()
group = matrix.groupby(['date_block_num'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num'], how='left')
 matrix['date_avg_item_cnt'] = matrix['date_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_avg_item_cnt')
 matrix.drop(['date_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
train_df['X'] = train_df.apply(lambda row: row['X'] if row['PlayDirection'] else 120-row['X'], axis=1)
 def new_orientation(angle, play_direction):
if play_direction == 0:
new_angle = 360.0 - angle
if new_angle == 360.0:
new_angle = 0.0
return new_angle
else:
return angle
train_df['Orientation'] = train_df.apply(lambda row: new_orientation(row['Orientation'], row['PlayDirection']), axis=1)
 train_df['Dir'] = train_df.apply(lambda row: new_orientation(row['Dir'], row['PlayDirection']), axis=1 )
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_cnt'] = matrix['date_item_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_item_avg_item_cnt')
 matrix.drop(['date_item_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
train_df['YardsLeft'] = train_df.apply(lambda row: 100 - row['YardLine'] if row['HomeField'] else row['YardLine'], axis=1)
 train_df['YardsLeft'] = train_df.apply(lambda row: row['YardsLeft'] if row['PlayDirection'] else 100 - row['YardsLeft'], axis=1)
 train_df.drop(train_df.index[(train_df['YardsLeft'] < train_df['Yards'])|(train_df['YardsLeft'] - 100 > train_df['Yards'])], inplace=True )
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_shop_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_avg_item_cnt'] = matrix['date_shop_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_shop_avg_item_cnt')
 matrix.drop(['date_shop_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
train_df = train_df.sort_values(by=['PlayId', 'Team', 'IsRusher', 'JerseyNumber'] ).reset_index()
train_df = train_df.drop(['index', 'IsRusher', 'Team'], axis=1)
 train_df = train_df.set_index(['GameId', 'PlayId'])
 cat_features = []
for col in train_df.columns:
if train_df[col].dtype =='object':
cat_features.append(col)
 train_df = train_df.drop(cat_features, axis=1)
 train_df.fillna(-999, inplace=True)
 clean_time = time.time()
print("Finish Data Cleaning, session time: {}s".format(clean_time - start_time))
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_cat_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_category_id'], how='left')
 matrix['date_cat_avg_item_cnt'] = matrix['date_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_cat_avg_item_cnt')
 matrix.drop(['date_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
def clean_data(df):
df['StadiumType'] = df['StadiumType'].apply(clean_StadiumType)
 df['StadiumType'] = df['StadiumType'].apply(transform_StadiumType)
 df['DefendersInTheBox_vs_Distance'] = df['DefendersInTheBox'] / df['Distance']
df['OffenseFormation'] = df['OffenseFormation'].apply(lambda x: x if x in off_form else np.nan)
 df = pd.concat([df.drop(['OffenseFormation'], axis=1), pd.get_dummies(df['OffenseFormation'], prefix='Formation')], axis=1)
 missing_cols = set(dummy_col)- set(df.columns)-set('Yards')
 for c in missing_cols:
df[c] = 0
df = df[dummy_col]
df['Turf'] = df['Turf'].map(Turf)
 df['Turf'] = df['Turf'] == 'Natural'
df['PossessionTeam'] = df['PossessionTeam'].map(map_abbr)
 df['HomeTeamAbbr'] = df['HomeTeamAbbr'].map(map_abbr)
 df['VisitorTeamAbbr'] = df['VisitorTeamAbbr'].map(map_abbr)
 df['HomePossesion'] = df['PossessionTeam'] == df['HomeTeamAbbr']
df['Field_eq_Possession'] = df['FieldPosition'] == df['PossessionTeam']
df['HomeField'] = df['FieldPosition'] == df['HomeTeamAbbr']
df['GameClock'] = df['GameClock'].apply(strtoseconds)
 df['PlayerHeight'] = df['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
df['PlayerBMI'] = 703*(df['PlayerWeight']/(df['PlayerHeight'])**2)
 df['TimeHandoff'] = df['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeSnap'] = df['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeDelta'] = df.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 df['PlayerBirthDate'] = df['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60*60*24*365.25
df['PlayerAge'] = df.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1)
 df['WindSpeed'] = df['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x)
 df['WindSpeed'] = df['WindSpeed'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 df['WindSpeed'] = df['WindSpeed'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x)
 df['WindSpeed'] = df['WindSpeed'].apply(lambda x: float(x)if str(x ).isnumeric() else -1)
 df['WindDirection'] = df['WindDirection'].apply(clean_WindDirection)
 df['WindDirection'] = df['WindDirection'].apply(transform_WindDirection)
 df['PlayDirection'] = df['PlayDirection'].apply(lambda x: x.strip() == 'right')
 df['Team'] = df['Team'].apply(lambda x: x.strip() =='home')
 indoor = "indoor"
df['GameWeather'] = df['GameWeather'].apply(lambda x: indoor if not pd.isna(x)and indoor in x else x)
 df['GameWeather'] = df['GameWeather'].apply(lambda x: x.lower().replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly' ).replace('clear and sunny', 'sunny and clear' ).replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 df['GameWeather'] = df['GameWeather'].apply(map_weather)
 df['IsRusher'] = df['NflId'] == df['NflIdRusher']
df['X'] = df.apply(lambda row: row['X'] if row['PlayDirection'] else 120-row['X'], axis=1)
 df['Orientation'] = df.apply(lambda row: new_orientation(row['Orientation'], row['PlayDirection']), axis=1)
 df['Dir'] = df.apply(lambda row: new_orientation(row['Dir'], row['PlayDirection']), axis=1)
 df['YardsLeft'] = df.apply(lambda row: 100-row['YardLine'] if row['HomeField'] else row['YardLine'], axis=1)
 df['YardsLeft'] = df.apply(lambda row: row['YardsLeft'] if row['PlayDirection'] else 100-row['YardsLeft'], axis=1)
 df = df.sort_values(by=['PlayId', 'Team', 'IsRusher', 'JerseyNumber'] ).reset_index()
df = df.drop(['TimeHandoff', 'TimeSnap', 'PlayerBirthDate', 'NflId', 'NflIdRusher', 'index', 'IsRusher', 'Team'], axis=1)
 df = df.set_index(['GameId', 'PlayId'])
 cat_features = []
for col in df.columns:
if df[col].dtype =='object':
cat_features.append(col)
 df = df.drop(cat_features, axis=1)
 df.fillna(-999, inplace=True)
 return df
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_cat_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'item_category_id'], how='left')
 matrix['date_shop_cat_avg_item_cnt'] = matrix['date_shop_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_cat_avg_item_cnt')
 matrix.drop(['date_shop_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
from sklearn.preprocessing import StandardScaler
import torch
import torch.nn as nn
import torchvision
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_type_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'type_code'], how='left')
 matrix['date_shop_type_avg_item_cnt'] = matrix['date_shop_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_type_avg_item_cnt')
 matrix.drop(['date_shop_type_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
def transform_df(df, is_train=True, col1=play_cols, col2=player_cols):
if is_train:
play_cols.remove('Yards')
 play_df = df[play_cols]
player_df = df[player_cols]
targets = df['Yards']
play_df = play_df[::22]
x1 = torch.tensor(StandardScaler().fit_transform(play_df ).astype(float), dtype=torch.float32)
 player_df = StandardScaler().fit_transform(player_df)
 player_df = player_df.reshape(play_df.shape[0], player_df.shape[1], 22)
 x2 = torch.tensor(player_df.astype(float), dtype=torch.float32)
 index = play_df.reset_index("GameId" ).index
y = torch.tensor(targets[::22].values, dtype=torch.float32)
 y_dis = torch.zeros([y.shape[0], 199])
 y_dis[torch.arange(y.shape[0]),(y + 99 ).long() ] = 1
return x1, x2, y_dis, index
else:
play_df = df[play_cols]
player_df = df[player_cols]
play_df = play_df[::22]
x1 = torch.tensor(StandardScaler().fit_transform(play_df ).astype(float), dtype=torch.float32)
 player_df = StandardScaler().fit_transform(player_df)
 player_df = player_df.reshape(play_df.shape[0], player_df.shape[1], 22)
 x2 = torch.tensor(player_df.astype(float), dtype=torch.float32)
 index = play_df.reset_index("GameId" ).index
return x1, x2, index
x1, x2, y, index = transform_df(train_df )
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_subtype_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'subtype_code'], how='left')
 matrix['date_shop_subtype_avg_item_cnt'] = matrix['date_shop_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_subtype_avg_item_cnt')
 matrix.drop(['date_shop_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>

 class MixtureNN(nn.Module):
def __init__(self):
super(MixtureNN, self ).__init__()
self.x1_dense = nn.Sequential(
 nn.Linear(len(play_cols), 200),
nn.ReLU() ,
nn.Linear(200, 100),
nn.ReLU() , 
)
 self.x2_conv = nn.Sequential(
 nn.Conv1d(in_channels=len(player_cols), out_channels=22, kernel_size=5, stride=1, padding=1),
nn.ReLU() ,
nn.Conv1d(in_channels=22, out_channels=11, kernel_size=3, stride=1, padding=1),
nn.ReLU() , 
)
 self.x_dense = nn.Sequential(
 nn.Linear(320, 480),
nn.BatchNorm1d(480),
nn.ReLU() ,
nn.Linear(480, 240),
nn.BatchNorm1d(240),
nn.ReLU() ,
nn.Linear(240, 199),
nn.Softmax(dim=1)
)
 def forward(self, x1, x2):
x1 = self.x1_dense(x1)
 x2 = self.x2_conv(x2 ).reshape(x1.shape[0], 220)
 x = torch.cat(( x1, x2), dim=1)
 x = self.x_dense(x)
 return x
ts = time.time()
group = matrix.groupby(['date_block_num', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'city_code'], how='left')
 matrix['date_city_avg_item_cnt'] = matrix['date_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_city_avg_item_cnt')
 matrix.drop(['date_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
MixtureNN()(x1,x2 ).shape
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'item_id', 'city_code'], how='left')
 matrix['date_item_city_avg_item_cnt'] = matrix['date_item_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_item_city_avg_item_cnt')
 matrix.drop(['date_item_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
before_training_time = time.time()
loss_func = CRPS()
def random_split(val_rate=0.2, batch=512):
total_index = np.arange(len(y))
val_index = np.random.choice(total_index, int(len(total_index)* 0.2), replace=False)
 train_index = np.delete(total_index, val_index)
 train_set = torch.utils.data.TensorDataset(x1[train_index], x2[train_index], y[train_index])
 val_set = torch.utils.data.TensorDataset(x1[val_index], x2[val_index], y[val_index])
 train_loader = torch.utils.data.DataLoader(train_set, batch_size=batch, shuffle=True, num_workers=8)
 val_loader = torch.utils.data.DataLoader(val_set, batch_size=batch, shuffle=True, num_workers=8)
 return train_loader, val_loader
def calculate_average_loss(dataloader: torch.utils.data.DataLoader):
running_loss = 0
for x1_batch, x2_batch, y_batch in dataloader:
prediction = model(x1_batch, x2_batch)
 loss = loss_func(prediction, y_batch)
 running_loss += float(loss)
 return running_loss / len(dataloader.dataset)
 print("======== START TRAINING...... ")
 cv_times = 0
models_params = []
sum_best_loss = 0
while cv_times < 20:
stable_step = 0
best_val_loss = 100
epoch = 0
session_time = time.time()
model = MixtureNN()
model.train()
optimizer = torch.optim.Adam(model.parameters() , lr=0.0001, weight_decay=0.01)
 print("======== Start Model {} ========".format(cv_times))
train_loader, val_loader = random_split(val_rate=0.3, batch=512)
 while stable_step < 50:
for x1_batch, x2_batch, y_batch in train_loader:
output = model(x1_batch, x2_batch)
 loss_val = loss_func(output, y_batch)
 loss_val.backward()
optimizer.step()
train_ave_loss = calculate_average_loss(train_loader)
 val_ave_loss = calculate_average_loss(val_loader)
 if val_ave_loss < best_val_loss:
best_val_loss = val_ave_loss
best_params = model.state_dict()
stable_step = 0
else:
stable_step += 1
epoch += 1
print("epoch: {}, train_loss: {}, val_loss: {}, best_val_loss: {}, stable_step: {}".format(
 epoch, train_ave_loss, val_ave_loss, best_val_loss, stable_step))
if epoch >= 500:
break
print("======== Finish Model {} ========".format(cv_times))
print("Model training time: {}s.The best evaluation loss is {}.".format(time.time() - session_time, best_val_loss))
models_params.append(best_params)
 sum_best_loss += best_val_loss
cv_times += 1
training_time = time.time()
print("======== END, total training time: {}s".format(training_time - before_training_time))
print("average_best_loss: {}".format(sum_best_loss / cv_times))
ts = time.time()
group = matrix.groupby(['date_block_num', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_type_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'type_code'], how='left')
 matrix['date_type_avg_item_cnt'] = matrix['date_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_type_avg_item_cnt')
 matrix.drop(['date_type_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
TOTAL: {}s".format(time.time() - init_time))
ts = time.time()
group = matrix.groupby(['date_block_num', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_subtype_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'subtype_code'], how='left')
 matrix['date_subtype_avg_item_cnt'] = matrix['date_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_subtype_avg_item_cnt')
 matrix.drop(['date_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<data_type_conversions>
def make_pred(df, sample, env, models_params):
clean_df = df
clean_df = clean_data(clean_df)
 x1, x2, index = transform_df(clean_df, is_train=False)
 y_preds = []
for params in models_params:
model.load_state_dict(params)
 model.eval()
y_preds.append(torch.cumsum(model(x1, x2), dim=1 ).detach().data.numpy())
 y_ave = np.array(y_preds ).mean(0)
 yardsleft = np.array(clean_df["YardsLeft"][::22])
 for i in range(len(yardsleft)) :
y_ave[i, :yardsleft[i] - 1] = 0
y_ave[i, yardsleft[i] + 99:] = 1
env.predict(pd.DataFrame(data=y_ave.clip(0, 1), columns=sample.columns))
return y_ave
ts = time.time()
group = train.groupby(['item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['item_id'], how='left')
 matrix['item_avg_item_price'] = matrix['item_avg_item_price'].astype(np.float16)
 group = train.groupby(['date_block_num','item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['date_item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_price'] = matrix['date_item_avg_item_price'].astype(np.float16)
 lags = [1,2,3,4,5,6]
matrix = lag_feature(matrix, lags, 'date_item_avg_item_price')
 for i in lags:
matrix['delta_price_lag_'+str(i)] = \ 
(matrix['date_item_avg_item_price_lag_'+str(i)] - matrix['item_avg_item_price'])/ matrix['item_avg_item_price']
def select_trend(row):
for i in lags:
if row['delta_price_lag_'+str(i)]:
return row['delta_price_lag_'+str(i)]
return 0
matrix['delta_price_lag'] = matrix.apply(select_trend, axis=1)
 matrix['delta_price_lag'] = matrix['delta_price_lag'].astype(np.float16)
 matrix['delta_price_lag'].fillna(0, inplace=True)
 fetures_to_drop = ['item_avg_item_price', 'date_item_avg_item_price']
for i in lags:
fetures_to_drop += ['date_item_avg_item_price_lag_'+str(i)]
fetures_to_drop += ['delta_price_lag_'+str(i)]
matrix.drop(fetures_to_drop, axis=1, inplace=True)
 time.time() - ts<merge>
for test, sample in tqdm.tqdm(env.iter_test()):
make_pred(test, sample, env, models_params)
ts = time.time()
group = train.groupby(['date_block_num','shop_id'] ).agg({'revenue': ['sum']})
 group.columns = ['date_shop_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_revenue'] = matrix['date_shop_revenue'].astype(np.float32)
 group = group.groupby(['shop_id'] ).agg({'date_shop_revenue': ['mean']})
 group.columns = ['shop_avg_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['shop_id'], how='left')
 matrix['shop_avg_revenue'] = matrix['shop_avg_revenue'].astype(np.float32)
 matrix['delta_revenue'] =(matrix['date_shop_revenue'] - matrix['shop_avg_revenue'])/ matrix['shop_avg_revenue']
matrix['delta_revenue'] = matrix['delta_revenue'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'delta_revenue')
 matrix.drop(['date_shop_revenue','shop_avg_revenue','delta_revenue'], axis=1, inplace=True)
 time.time() - ts<feature_engineering>
env.write_submission_file()
<categorify><EOS> 
total_time = time.time()
print('Total running time is: {}s'.format(total_time - start_time))